{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Execution time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We compare **performance** (execution time) of different CLBlast GEMM implementations (e.g. original vs Lift overlay) on the row-major and column-major layouts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Includes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scientific"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import IPython as ip\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import matplotlib as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('IPython version: %s' % ip.__version__)\n",
    "print('NumPy version: %s' % np.__version__)\n",
    "print('SciPy version: %s' % sp.__version__)\n",
    "print('Pandas version: %s' % pd.__version__)\n",
    "print('Matplotlib version: %s' % mp.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "%matplotlib inline\n",
    "# import scipy.stats as st"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collective Knowledge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import ck.kernel as ck\n",
    "print('CK version: %s' % ck.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Access experimental results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def get_experimental_results(tags):\n",
    "    search = 'search'\n",
    "    repo_uoa = 'local'\n",
    "    module_uoa = 'experiment'\n",
    "    r=ck.access({'action':search, 'module_uoa':module_uoa, 'tags':tags})\n",
    "    if r['return']>0:\n",
    "        print (\"Error: %s\" % r['error'])\n",
    "        exit(1)\n",
    "    experiments=r['lst']\n",
    "    \n",
    "    dfs = []\n",
    "    for experiment in experiments:\n",
    "        data_uoa = experiment['data_uoa']\n",
    "        r = ck.access({'action':'list_points', 'repo_oua':repo_uoa, 'module_uoa':module_uoa, 'data_uoa':data_uoa})\n",
    "        if r['return']>0:\n",
    "            print (\"Error: %s\" % r['error'])\n",
    "            exit(1)\n",
    "        path = r['path']\n",
    "        points = r['points']\n",
    "        for point in points:\n",
    "            with open(os.path.join(path, 'ckp-%s.0001.json' % point)) as point_file:\n",
    "                point_data_raw = json.load(point_file)\n",
    "            # DataFrame columns.\n",
    "            characteristics = [\n",
    "                {\n",
    "                    'time (ms)'   : np.float32(characteristics['run'].get('ms_1',[0])[0]),\n",
    "                    'success?'    : characteristics['run'].get('run_success','n/a'),\n",
    "                    'dvdt_prof'   : characteristics['run'].get('dvdt_prof',[])\n",
    "#                    'layout'      : characteristics['run'].get('layout','n/a')\n",
    "                }\n",
    "                for characteristics in point_data_raw['characteristics_list'] \n",
    "                if characteristics['run'].get('run_success','')!=''\n",
    "            ]\n",
    "            df = pd.DataFrame(characteristics)\n",
    "            df.columns.name = 'run characteristic'\n",
    "            df.index.name = 'repetition'\n",
    "            # DataFrame indices.\n",
    "            df['program'] = point_data_raw['choices']['data_uoa']\n",
    "            layout_info = point_data_raw['choices']['env']['CLBLAST_LAYOUT']\n",
    "            if layout_info == 101: df['layout'] = 'row-major'\n",
    "            elif layout_info == 102: df['layout'] = 'column-major'\n",
    "            else: df['layout'] = 'unknown'\n",
    "            df = df.set_index(['program','layout'], append=True)\n",
    "            df = df.reorder_levels(('program', 'layout', 'repetition'))\n",
    "            dfs.append(df)\n",
    "    results = pd.concat(dfs)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results = get_experimental_results('gemmbench,xgemm,clblast')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show execution time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results[['time (ms)']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot execution time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot(mean, std):\n",
    "    mean \\\n",
    "        .plot(yerr=std, title='Execution time (ms)', kind='bar', colormap=cm.autumn,\n",
    "            figsize=[16, 8], rot=0, grid=True, legend=True) \\\n",
    "        .legend(loc='upper left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare row-major layout vs column-major layout for each program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mean = results['time (ms)'].groupby(level=['program', 'layout']).mean().unstack('layout')\n",
    "std = results['time (ms)'].groupby(level=['program', 'layout']).std().unstack('layout')\n",
    "plot(mean, std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare CLBlast vs Lift overlay for each layout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mean = results['time (ms)'].groupby(level=['program', 'layout']).mean().unstack('program')\n",
    "std = results['time (ms)'].groupby(level=['program', 'layout']).std().unstack('program')\n",
    "plot(mean, std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show profiling info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Pick the first repetition of the first experiment for now.\n",
    "trace = results['dvdt_prof'].iloc[0]\n",
    "if not trace:\n",
    "    raise Exception(\"No OpenCL profiling information!\")\n",
    "# What's that experiment, by the way?\n",
    "results['dvdt_prof'].index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "r=ck.access({'action':'show', 'module_uoa':'env', 'tags':'tool,opencl,dvdt,prof'})\n",
    "if r['return']>0:\n",
    "    print (\"Error: %s\" % r['error'])\n",
    "    exit(1)\n",
    "# Get path the first returned environment entry.\n",
    "dvdt_prof_dir=r['lst'][0]['meta']['env']['CK_ENV_TOOL_DVDT_PROF']\n",
    "dvdt_prof_src_python=os.path.join(dvdt_prof_dir,'src','python')\n",
    "sys.path.append(dvdt_prof_src_python)\n",
    "import prof_wrangler as pw\n",
    "pw.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trace = pw.index_calls(trace)\n",
    "unit = 'ms'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kernel enqueues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Partial trace only containing kernel enqueues.\n",
    "kernel_enqueues = pw.filter_calls(trace, ['clEnqueueNDRangeKernel'])\n",
    "# Kernel enqueues as a DataFrame.\n",
    "df_kernel_enqueues = pw.df_kernel_enqueues(kernel_enqueues, unit)\n",
    "df_kernel_enqueues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df_kernel_enqueues.info(memory_usage=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_enqueues_total = len(kernel_enqueues)\n",
    "num_enqueues_per_repetition = 4\n",
    "df_kernel_enqueues['kernel_index'] = (pd.Series(range(num_enqueues_total)) % num_enqueues_per_repetition).values\n",
    "df_kernel_enqueues = df_kernel_enqueues \\\n",
    "    .set_index('kernel_index', append=True) \\\n",
    "    .reorder_levels(['call_index','kernel_index','name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_kernel_enqueues_stats = df_kernel_enqueues.groupby(level='kernel_index').describe()\n",
    "df_kernel_enqueues_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "kernel_time = \\\n",
    "    df_kernel_enqueues_stats[['p3 - p2 (%s)' % unit,'p2 - p1 (%s)' % unit, 'p1 - p0 (%s)' % unit]] \\\n",
    "    .unstack('kernel_index') \\\n",
    "    .rename(columns={0:'transpose A', 1:'transpose B', 2:'Xgemm', 3:'transpose C'}) \\\n",
    "    .stack()\n",
    "kernel_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mean=kernel_time.ix['mean']\n",
    "std=kernel_time.ix['std']\n",
    "mean.plot(yerr=std, title='Execution time (%s)' % unit,\n",
    "          kind='bar', stacked=True,\n",
    "          figsize=[16, 8], rot=45,\n",
    "          grid=True, legend=True,\n",
    "          colormap=cm.autumn) \\\n",
    "    .set_xlabel('Kernel')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}