zrepl_patched/zfs/zfscmd/zfscmd-logging-scraper/analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "\n",
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ./parsed.json is the stdout of the scraper tool in this directory\n",
    "df = pd.read_json(\"./parsed.json\", lines=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_ds(entity):\n",
    "    m = re.search(r\"(?P<dataset>[^@#]*)([@#].+)?\", entity)\n",
    "    return m.group(\"dataset\")\n",
    "    \n",
    "def parse_cmd(row):\n",
    "    cmd  = row.Cmd\n",
    "    binary, verb, *tail = re.split(r\"\\s+\", cmd) # NOTE whitespace in dataset names => don't use comp\n",
    "    \n",
    "    dataset = None\n",
    "    if binary == \"zfs\":\n",
    "        if verb == \"send\":            \n",
    "            dataset = parse_ds(tail[-1])\n",
    "            if \"-n\" in tail:\n",
    "                verb = \"send-dry\"\n",
    "        elif verb == \"recv\" or verb == \"receive\":\n",
    "            verb = \"receive\"\n",
    "            if len(tail) > 0:\n",
    "                dataset = parse_ds(tail[-1])\n",
    "            else:\n",
    "                verb = \"receive-CLI-test\"\n",
    "        elif verb == \"get\":\n",
    "            dataset = parse_ds(tail[-1])\n",
    "        elif verb == \"list\":\n",
    "            if \"-r\" in tail and \"-d\" in tail and \"1\" in tail:\n",
    "                dataset = parse_ds(tail[-1])\n",
    "                verb = \"list-single-dataset\"\n",
    "            else:\n",
    "                dataset = \"!ALL_POOLS!\"\n",
    "                verb = \"list-all-filesystems\"\n",
    "        elif verb == \"bookmark\":\n",
    "            dataset = parse_ds(tail[-2])\n",
    "        elif verb == \"hold\":\n",
    "            dataset = parse_ds(tail[-1])\n",
    "        elif verb == \"snapshot\":\n",
    "            dataset = parse_ds(tail[-1])\n",
    "        elif verb == \"release\":\n",
    "            dss = tail[-1].split(\",\")\n",
    "            if len(dss) > 1:\n",
    "                raise Exception(\"cannot handle batch-release\")\n",
    "            dataset = parse_ds(dss[0])\n",
    "        elif verb == \"holds\" and \"-H\" in tail:\n",
    "            dss = tail[-1].split(\",\")\n",
    "            if len(dss) > 1:\n",
    "                raise Exception(\"cannot handle batch-holds\")\n",
    "            dataset = parse_ds(dss[0])\n",
    "        elif verb == \"destroy\":\n",
    "            dss = tail[-1].split(\",\")\n",
    "            if len(dss) > 1:\n",
    "                raise Exception(\"cannot handle batch-holds\")\n",
    "            dataset = parse_ds(dss[0])\n",
    "    \n",
    "    return {'action':binary + \"-\" + verb, 'dataset': dataset }\n",
    "    \n",
    "    \n",
    "res = df.apply(parse_cmd, axis='columns', result_type='expand')\n",
    "res = pd.concat([df, res], axis='columns')\n",
    "for cat in [\"action\", \"dataset\"]:\n",
    "    res[cat] = res[cat].astype('category')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res[\"OtherTime\"] = res.TotalTime - res.Usertime - res.Systime\n",
    "x = res.melt(id_vars=[\"action\", \"dataset\"], value_vars=[\"TotalTime\", \"OtherTime\", \"Usertime\", \"Systime\"])\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"commands with NaN values\")\n",
    "set(x[x.isna().any(axis=1)].action.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (~x.action.astype('str').isin([\"zfs-send\", \"zfs-recv\"]))\n",
    "totaltimes = x[(x.variable == \"TotalTime\")].groupby([\"action\", \"dataset\"]).sum().reset_index()\n",
    "display(totaltimes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "totaltimes_by_action = totaltimes.groupby(\"action\").sum().sort_values(by=\"value\")\n",
    "totaltimes_by_action.plot.barh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "totaltimes.groupby(\"dataset\").sum().plot.barh(fontsize=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_expensive_action = totaltimes_by_action.idxmax().value\n",
    "display(most_expensive_action)\n",
    "most_expensive_action_by_dataset = totaltimes[totaltimes.action == most_expensive_action].groupby(\"dataset\").sum().sort_values(by=\"value\")\n",
    "most_expensive_action_by_dataset.plot.barh(rot=50, fontsize=5, figsize=(10, 20))\n",
    "plt.savefig('most-expensive-command.pdf')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}