{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Compare performance of different CNN architectures" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We compare **performance**:\n", "- of 4 different CNN **models** (net + weights):\n", " - GoogleNet;\n", " - AlexNet;\n", " - SqueezeNet 1.0;\n", " - SqueezeNet 1.1;\n", "\n", "- when using 4 different **libraries**:\n", "\n", " - [CPU] clblast_mali_overlay 0.2.18;\n", " - [GPU] clBLAS 2.4;\n", " - [GPU] CLBlast dev (> 0.8.0);\n", " - [GPU] CLBlast dev (> 0.8.0) with Mali-optimised overlay." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Includes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Standard" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import os\n", "import sys\n", "import json" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Scientific" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import IPython as ip\n", "import numpy as np\n", "import scipy as sp\n", "import pandas as pd\n", "import matplotlib as mp" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "print 'IPython version: %s' % ip.__version__\n", "print 'NumPy version: %s' % np.__version__\n", "print 'SciPy version: %s' % sp.__version__\n", "print 'Pandas version: %s' % pd.__version__\n", "print 'Matplotlib version: %s' % mp.__version__" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "from matplotlib import cm\n", "%matplotlib inline\n", "# import scipy.stats as st" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Collective Knowledge" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import ck.kernel as ck\n", "print 'CK version: %s' % ck.__version__" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Access experimental results" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "def search_experimental_points_by_tags(tags):\n", " r=ck.access({'action':'search', 'module_uoa':'experiment', 'tags':tags})\n", " if r['return']>0:\n", " print (\"Error: %s\" % r['error'])\n", " exit(1)\n", " # FIXME: For now, assume a single entry per the given tags.\n", " r=ck.access({'action':'list_points', 'module_uoa':'experiment', 'data_uoa': r['lst'][0]['data_uoa']})\n", " if r['return']>0:\n", " print (\"Error: %s\" % r['error'])\n", " exit(1)\n", " \n", " results = {}\n", " for point in r['points']:\n", " with open(os.path.join(r['path'], 'ckp-%s.0001.json' % point)) as point_file:\n", " point_data_raw = json.load(point_file)\n", " point_data_dict = {}\n", " time_fw_ms = [\n", " characteristics['run']['time_fw_ms'] \n", " for characteristics in point_data_raw['characteristics_list']\n", " if characteristics['run']['run_success'] == 'yes'\n", " ]\n", " batch_size = point_data_raw['choices']['env']['CK_CAFFE_BATCH_SIZE']\n", " results[batch_size] = time_fw_ms\n", " return results\n", "\n", "def get_min_time_per_image(results):\n", " df = pd.DataFrame(data=results)\n", " df.index.name = 'repetition'\n", " df.columns.name = 'batch size'\n", " return (df.describe().ix['min'] / range(1,len(results)+1)).min()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### AlexNet" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "alexnet_clblas_tags = 'time,caffemodel,alexnet,clblas'\n", "alexnet_clblas_results = search_experimental_points_by_tags(alexnet_clblas_tags)\n", "alexnet_clblas_min_time_per_image = get_min_time_per_image(alexnet_clblas_results)\n", "alexnet_clblas_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "alexnet_clblast_development_tags = 'time,caffemodel,alexnet,clblast,vdevelopment'\n", "alexnet_clblast_development_results = search_experimental_points_by_tags(alexnet_clblast_development_tags)\n", "alexnet_clblast_development_min_time_per_image = get_min_time_per_image(alexnet_clblast_development_results)\n", "alexnet_clblast_development_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "alexnet_clblast_mali_overlay_tags = 'time,caffemodel,alexnet,clblast,vmali-overlay'\n", "alexnet_clblast_mali_overlay_results = search_experimental_points_by_tags(alexnet_clblast_mali_overlay_tags)\n", "alexnet_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(alexnet_clblast_mali_overlay_results)\n", "alexnet_clblast_mali_overlay_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "alexnet_openblas_tags = 'time,caffemodel,alexnet,openblas'\n", "alexnet_openblas_results = search_experimental_points_by_tags(alexnet_openblas_tags)\n", "alexnet_openblas_min_time_per_image = get_min_time_per_image(alexnet_openblas_results)\n", "alexnet_openblas_min_time_per_image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### SqueezeNet 1.0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_0_clblas_tags = 'time,caffemodel,squeezenet-1.0,clblas'\n", "squeezenet_1_0_clblas_results = search_experimental_points_by_tags(squeezenet_1_0_clblas_tags)\n", "squeezenet_1_0_clblas_min_time_per_image = get_min_time_per_image(squeezenet_1_0_clblas_results)\n", "squeezenet_1_0_clblas_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_0_clblast_development_tags = 'time,caffemodel,squeezenet-1.0,clblast,vdevelopment'\n", "squeezenet_1_0_clblast_development_results = search_experimental_points_by_tags(squeezenet_1_0_clblast_development_tags)\n", "squeezenet_1_0_clblast_development_min_time_per_image = get_min_time_per_image(squeezenet_1_0_clblast_development_results)\n", "squeezenet_1_0_clblast_development_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_0_clblast_mali_overlay_tags = 'time,caffemodel,squeezenet-1.0,clblast,vmali-overlay'\n", "squeezenet_1_0_clblast_mali_overlay_results = search_experimental_points_by_tags(squeezenet_1_0_clblast_mali_overlay_tags)\n", "squeezenet_1_0_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(squeezenet_1_0_clblast_mali_overlay_results)\n", "squeezenet_1_0_clblast_mali_overlay_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_0_openblas_tags = 'time,caffemodel,squeezenet-1.0,openblas'\n", "squeezenet_1_0_openblas_results = search_experimental_points_by_tags(squeezenet_1_0_openblas_tags)\n", "squeezenet_1_0_openblas_min_time_per_image = get_min_time_per_image(squeezenet_1_0_openblas_results)\n", "squeezenet_1_0_openblas_min_time_per_image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### SqueezeNet 1.1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_1_clblas_tags = 'time,caffemodel,squeezenet-1.1,clblas'\n", "squeezenet_1_1_clblas_results = search_experimental_points_by_tags(squeezenet_1_1_clblas_tags)\n", "squeezenet_1_1_clblas_min_time_per_image = get_min_time_per_image(squeezenet_1_1_clblas_results)\n", "squeezenet_1_1_clblas_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_1_clblast_development_tags = 'time,caffemodel,squeezenet-1.1,clblast,vdevelopment'\n", "squeezenet_1_1_clblast_development_results = search_experimental_points_by_tags(squeezenet_1_1_clblast_development_tags)\n", "squeezenet_1_1_clblast_development_min_time_per_image = get_min_time_per_image(squeezenet_1_1_clblast_development_results)\n", "squeezenet_1_1_clblast_development_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_1_clblast_mali_overlay_tags = 'time,caffemodel,squeezenet-1.1,clblast,vmali-overlay'\n", "squeezenet_1_1_clblast_mali_overlay_results = search_experimental_points_by_tags(squeezenet_1_1_clblast_mali_overlay_tags)\n", "squeezenet_1_1_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(squeezenet_1_1_clblast_mali_overlay_results)\n", "squeezenet_1_1_clblast_mali_overlay_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "squeezenet_1_1_openblas_tags = 'time,caffemodel,squeezenet-1.1,openblas'\n", "squeezenet_1_1_openblas_results = search_experimental_points_by_tags(squeezenet_1_1_openblas_tags)\n", "squeezenet_1_1_openblas_min_time_per_image = get_min_time_per_image(squeezenet_1_1_openblas_results)\n", "squeezenet_1_1_openblas_min_time_per_image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### GoogleNet" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "googlenet_clblas_tags = 'time,caffemodel,googlenet,clblas'\n", "googlenet_clblas_results = search_experimental_points_by_tags(googlenet_clblas_tags)\n", "googlenet_clblas_min_time_per_image = get_min_time_per_image(googlenet_clblas_results)\n", "googlenet_clblas_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "googlenet_clblast_development_tags = 'time,caffemodel,googlenet,clblast,vdevelopment'\n", "googlenet_clblast_development_results = search_experimental_points_by_tags(googlenet_clblast_development_tags)\n", "googlenet_clblast_development_min_time_per_image = get_min_time_per_image(googlenet_clblast_development_results)\n", "googlenet_clblast_development_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "googlenet_clblast_mali_overlay_tags = 'time,caffemodel,googlenet,clblast,vmali-overlay'\n", "googlenet_clblast_mali_overlay_results = search_experimental_points_by_tags(googlenet_clblast_mali_overlay_tags)\n", "googlenet_clblast_mali_overlay_min_time_per_image = get_min_time_per_image(googlenet_clblast_mali_overlay_results)\n", "googlenet_clblast_mali_overlay_min_time_per_image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "googlenet_openblas_tags = 'time,caffemodel,googlenet,openblas'\n", "googlenet_openblas_results = search_experimental_points_by_tags(googlenet_openblas_tags)\n", "googlenet_openblas_min_time_per_image = get_min_time_per_image(googlenet_openblas_results)\n", "googlenet_openblas_min_time_per_image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Data frame" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "data = {\n", " 'AlexNet' : {\n", " 'OpenBLAS: v0.2.18' : alexnet_openblas_min_time_per_image,\n", " 'clBLAS: v2.4' : alexnet_clblas_min_time_per_image,\n", " 'CLBlast: development' : alexnet_clblast_development_min_time_per_image,\n", " 'CLBlast: Mali overlay' : alexnet_clblast_mali_overlay_min_time_per_image\n", " },\n", "# 'GoogleNet' : {\n", "# 'OpenBLAS: v0.2.18' : googlenet_openblas_min_time_per_image,\n", "# 'clBLAS: v2.4' : googlenet_clblas_min_time_per_image,\n", "# 'CLBlast: development' : googlenet_clblast_development_min_time_per_image,\n", "# 'CLBlast: Mali overlay' : googlenet_clblast_mali_overlay_min_time_per_image\n", "# },\n", " 'SqueezeNet 1.0' : {\n", " 'OpenBLAS: v0.2.18' : squeezenet_1_0_openblas_min_time_per_image,\n", " 'clBLAS: v2.4' : squeezenet_1_0_clblas_min_time_per_image,\n", " 'CLBlast: development' : squeezenet_1_0_clblast_development_min_time_per_image,\n", " 'CLBlast: Mali overlay' : squeezenet_1_0_clblast_mali_overlay_min_time_per_image\n", " },\n", " 'SqueezeNet 1.1' : {\n", " 'OpenBLAS: v0.2.18' : squeezenet_1_1_openblas_min_time_per_image,\n", " 'clBLAS: v2.4' : squeezenet_1_1_clblas_min_time_per_image,\n", " 'CLBlast: development' : squeezenet_1_1_clblast_development_min_time_per_image,\n", " 'CLBlast: Mali overlay' : squeezenet_1_1_clblast_mali_overlay_min_time_per_image\n", " },\n", "}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df = pd.DataFrame(data)\n", "df" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Plot" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df.T \\\n", " .plot(title='Execution time per image (ms)',\n", " kind='bar', rot=0, ylim=[0,1100], figsize=[12, 6], grid=True, legend=True, colormap=cm.autumn)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "df \\\n", " .plot(title='Execution time per image (ms)',\n", " kind='bar', rot=0, ylim=[0,1100], figsize=[12, 6], grid=True, legend=True, colormap=cm.autumn)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.12" } }, "nbformat": 4, "nbformat_minor": 0 }