HAR_LOSO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Smart Wearables -- Bonsai Tree Classification on stream data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Introduction & Library Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score\n",
    "import scipy.stats as st\n",
    "import sys\n",
    "from sklearn.model_selection import train_test_split\n",
    "from bonsai.base.regtree import RegTree\n",
    "from bonsai.base.alphatree import AlphaTree\n",
    "from bonsai.base.c45tree import C45Tree\n",
    "from bonsai.base.ginitree import GiniTree\n",
    "from bonsai.base.xgbtree import XGBTree\n",
    "from bonsai.base.friedmantree import FriedmanTree\n",
    "from bonsai.ensemble.randomforests import RandomForests\n",
    "from bonsai.ensemble.paloboost import PaloBoost\n",
    "from bonsai.ensemble.gbm import GBM\n",
    "#from bonsai.ensemble.streamforests import StreamForests\n",
    "import copy\n",
    "import sys\n",
    "import json\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from keras import optimizers\n",
    "from utils import *\n",
    "from model import *\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "# Setting seed for reproducability\n",
    "np.random.seed(1234)  \n",
    "PYTHONHASHSEED = 0\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import confusion_matrix, recall_score, precision_score\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, LSTM, Activation\n",
    "%matplotlib inline\n",
    "\n",
    "#import pydot\n",
    "#import graphviz\n",
    "#pydot.find_graphviz = lambda: True\n",
    "import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Gathering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_input_file = 'data/LOSO/USCHAD.npz'\n",
    "np_load_old = np.load\n",
    "\n",
    "# modify the default parameters of np.load\n",
    "np.load = lambda *a,**k: np_load_old(*a, allow_pickle=True, **k)\n",
    "tmp = np.load(data_input_file)\n",
    "np.load = np_load_old"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tmp['X']\n",
    "X = X[:, 0, :, :]\n",
    "y = tmp['y']\n",
    "folds = tmp['folds']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hancrafted Template 2017 data/LOSO/USCHAD.npz\n"
     ]
    }
   ],
   "source": [
    "n_class = y.shape[1]\n",
    "y = np.argmax(y, axis=1)\n",
    "print('Hancrafted Template 2017 {}'.format(data_input_file))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def A(sample):\n",
    "    feat = []\n",
    "    for col in range(0,sample.shape[1]):\n",
    "        average = np.average(sample[:,col])\n",
    "        feat.append(average)\n",
    "\n",
    "    return feat\n",
    "\n",
    "def SD(sample):\n",
    "    feat = []\n",
    "    for col in range(0, sample.shape[1]):\n",
    "        std = np.std(sample[:, col])\n",
    "        feat.append(std)\n",
    "\n",
    "    return feat\n",
    "\n",
    "def AAD(sample):\n",
    "    feat = []\n",
    "    for col in range(0, sample.shape[1]):\n",
    "        data = sample[:, col]\n",
    "        add = np.mean(np.absolute(data - np.mean(data)))\n",
    "        feat.append(add)\n",
    "\n",
    "    return feat\n",
    "\n",
    "def ARA(sample):\n",
    "    #Average Resultant Acceleration[1]:\n",
    "    # Average of the square roots of the sum of the values of each axis squared √(xi^2 + yi^2+ zi^2) over the ED\n",
    "    feat = []\n",
    "    sum_square = 0\n",
    "    sample = np.power(sample, 2)\n",
    "    for col in range(0, sample.shape[1]):\n",
    "        sum_square = sum_square + sample[:, col]\n",
    "\n",
    "    sample = np.sqrt(sum_square)\n",
    "    average = np.average(sample)\n",
    "    feat.append(average)\n",
    "    return feat\n",
    "\n",
    "def TBP(sample):\n",
    "    from scipy import signal\n",
    "    feat = []\n",
    "    sum_of_time = 0\n",
    "    for col in range(0, sample.shape[1]):\n",
    "        data = sample[:, col]\n",
    "        peaks = signal.find_peaks_cwt(data, np.arange(1,4))\n",
    "\n",
    "        feat.append(peaks)\n",
    "\n",
    "    return feat\n",
    "\n",
    "def feature_extraction(X):\n",
    "    #Extracts the features, as mentioned by Catal et al. 2015\n",
    "    # Average - A,\n",
    "    # Standard Deviation - SD,\n",
    "    # Average Absolute Difference - AAD,\n",
    "    # Average Resultant Acceleration - ARA(1),\n",
    "    # Time Between Peaks - TBP\n",
    "    X_tmp = []\n",
    "    for sample in X:\n",
    "        features = A(copy.copy(sample))\n",
    "        features = np.hstack((features, A(copy.copy(sample))))\n",
    "        features = np.hstack((features, SD(copy.copy(sample))))\n",
    "        features = np.hstack((features, AAD(copy.copy(sample))))\n",
    "        features = np.hstack((features, ARA(copy.copy(sample))))\n",
    "        #features = np.hstack((features, TBP(sample)))\n",
    "        X_tmp.append(features)\n",
    "\n",
    "    X = np.array(X_tmp)\n",
    "    return X\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. RegTree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy[0.5207] Recall[0.6025] F1[0.4862] at fold[0]\n",
      "______________________________________________________\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-f23e12d2a73b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;31m#X_train=X_train.transpose(0,1,2).reshape(X_train.shape[0],-1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;31m#X_test=X_test.transpose(0,1,2).reshape(X_test.shape[0],-1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeature_extraction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeature_extraction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-84957f613280>\u001b[0m in \u001b[0;36mfeature_extraction\u001b[0;34m(X)\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAAD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mARA\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0;31m#features = np.hstack((features, TBP(sample)))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/copy.py\u001b[0m in \u001b[0;36mcopy\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0mcopier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"__copy__\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcopier\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcopier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0mreductor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdispatch_table\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "avg_acc = []\n",
    "avg_recall = []\n",
    "avg_f1 = []\n",
    "avg_ttime=[]\n",
    "avg_ptime=[]\n",
    "avg_size=[]\n",
    "for i in range(0, len(folds)):\n",
    "    train_idx = folds[i][0]\n",
    "    test_idx = folds[i][1]\n",
    "\n",
    "    X_train, y_train = X[train_idx], y[train_idx]\n",
    "    X_test, y_test = X[test_idx], y[test_idx]\n",
    "\n",
    "        #Your train goes here. For instance:\n",
    "    #X_train=X_train.transpose(0,1,2).reshape(X_train.shape[0],-1)\n",
    "    #X_test=X_test.transpose(0,1,2).reshape(X_test.shape[0],-1)\n",
    "    X_train = feature_extraction(X_train)\n",
    "    X_test = feature_extraction(X_test)      \n",
    "        \n",
    "    method = RegTree(max_depth=400,min_samples_split=1,min_samples_leaf=2)\n",
    "    t0=time.time()\n",
    "    method.fit(X_train, y_train)\n",
    "    avg_ttime.append(time.time()-t0)\n",
    "            #Your testing goes here. For instance:\n",
    "    t1=time.time()\n",
    "    y_pred = method.predict(X_test)\n",
    "    avg_ptime.append(time.time()-t1)\n",
    "    y_pred=np.round(y_pred,0)\n",
    "    y_pred=y_pred.astype(int)\n",
    "    v=method.dump()\n",
    "    avg_size.append(round(v.__sizeof__()/1024,3))\n",
    "\n",
    "    acc_fold = accuracy_score(y_test, y_pred)\n",
    "    avg_acc.append(acc_fold)\n",
    "\n",
    "    recall_fold = recall_score(y_test, y_pred, average='macro')\n",
    "    avg_recall.append(recall_fold)\n",
    "\n",
    "    f1_fold  = f1_score(y_test, y_pred, average='macro')\n",
    "    avg_f1.append(f1_fold)\n",
    "\n",
    "    print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.format(acc_fold, recall_fold, f1_fold ,i))\n",
    "    print('______________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Accuracy[0.5000] IC [0.4380, 0.5620]\n",
      "Mean Recall[0.4764] IC [0.4352, 0.5176]\n",
      "Mean F1[0.4137] IC [0.3679, 0.4596]\n",
      "Mean size[2.448]\n",
      "Mean training time[1057.860]\n",
      "Mean prediction time[0.184]\n"
     ]
    }
   ],
   "source": [
    "ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc))\n",
    "ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall))\n",
    "ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1))\n",
    "print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_acc), ic_acc[0], ic_acc[1]))\n",
    "print('Mean Recall[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_recall), ic_recall[0], ic_recall[1]))\n",
    "print('Mean F1[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_f1), ic_f1[0], ic_f1[1]))\n",
    "print('Mean size[{:.3f}]'.format(np.mean(avg_size)))\n",
    "print('Mean training time[{:.3f}]'.format(round(np.mean(avg_ttime)*1000,3)))\n",
    "print('Mean prediction time[{:.3f}]'.format(round(np.mean(avg_ptime)*1000,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. XGBTree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy[1.0000] Recall[1.0000] F1[1.0000] at fold[0]\n",
      "______________________________________________________\n",
      "Accuracy[0.8403] Recall[0.8579] F1[0.8405] at fold[1]\n",
      "______________________________________________________\n",
      "Accuracy[0.9442] Recall[0.9476] F1[0.9444] at fold[2]\n",
      "______________________________________________________\n",
      "Accuracy[0.7820] Recall[0.7721] F1[0.7517] at fold[3]\n",
      "______________________________________________________\n",
      "Accuracy[0.8340] Recall[0.8406] F1[0.7990] at fold[4]\n",
      "______________________________________________________\n",
      "Accuracy[0.9129] Recall[0.9167] F1[0.8930] at fold[5]\n",
      "______________________________________________________\n",
      "Accuracy[0.8182] Recall[0.8261] F1[0.7740] at fold[6]\n",
      "______________________________________________________\n",
      "Accuracy[0.8452] Recall[0.8659] F1[0.8334] at fold[7]\n",
      "______________________________________________________\n",
      "Accuracy[0.9721] Recall[0.9716] F1[0.9711] at fold[8]\n",
      "______________________________________________________\n",
      "Accuracy[0.9395] Recall[0.9453] F1[0.9360] at fold[9]\n",
      "______________________________________________________\n"
     ]
    }
   ],
   "source": [
    "avg_acc = []\n",
    "avg_recall = []\n",
    "avg_f1 = []\n",
    "avg_ttime=[]\n",
    "avg_ptime=[]\n",
    "avg_size=[]\n",
    "for i in range(0, len(folds)):\n",
    "    train_idx = folds[i][0]\n",
    "    test_idx = folds[i][1]\n",
    "\n",
    "    X_train, y_train = X[train_idx], y[train_idx]\n",
    "    X_test, y_test = X[test_idx], y[test_idx]\n",
    "    \n",
    "    X_train = feature_extraction(X_train)\n",
    "    X_test = feature_extraction(X_test)      \n",
    "        \n",
    "    method = XGBTree(max_depth=10,min_samples_split=1,min_samples_leaf=1)\n",
    "    t0=time.time()\n",
    "    method.fit(X_train, y_train)\n",
    "    avg_ttime.append(time.time()-t0)\n",
    "            #Your testing goes here. For instance:\n",
    "    t1=time.time()\n",
    "    y_pred = method.predict(X_test)\n",
    "    avg_ptime.append(time.time()-t1)\n",
    "    y_pred=np.round(y_pred,0)\n",
    "    y_pred=y_pred.astype(int)\n",
    "    v=method.dump()\n",
    "    avg_size.append(round(v.__sizeof__()/1024,3))\n",
    "\n",
    "    acc_fold = accuracy_score(y_test, y_pred)\n",
    "    avg_acc.append(acc_fold)\n",
    "\n",
    "    recall_fold = recall_score(y_test, y_pred, average='macro')\n",
    "    avg_recall.append(recall_fold)\n",
    "\n",
    "    f1_fold  = f1_score(y_test, y_pred, average='macro')\n",
    "    avg_f1.append(f1_fold)\n",
    "\n",
    "    print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.format(acc_fold, recall_fold, f1_fold ,i))\n",
    "    print('______________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Accuracy[0.8888] IC [0.8460, 0.9317]\n",
      "Mean Recall[0.8944] IC [0.8522, 0.9366]\n",
      "Mean F1[0.8743] IC [0.8240, 0.9246]\n",
      "Mean size[0.945]\n",
      "Mean training time[1350.741]\n",
      "Mean prediction time[0.123]\n"
     ]
    }
   ],
   "source": [
    "ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc))\n",
    "ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall))\n",
    "ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1))\n",
    "print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_acc), ic_acc[0], ic_acc[1]))\n",
    "print('Mean Recall[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_recall), ic_recall[0], ic_recall[1]))\n",
    "print('Mean F1[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_f1), ic_f1[0], ic_f1[1]))\n",
    "print('Mean size[{:.3f}]'.format(np.mean(avg_size)))\n",
    "print('Mean training time[{:.3f}]'.format(round(np.mean(avg_ttime)*1000,3)))\n",
    "print('Mean prediction time[{:.3f}]'.format(round(np.mean(avg_ptime)*1000,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. FriedmanTree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy[1.0000] Recall[1.0000] F1[1.0000] at fold[0]\n",
      "______________________________________________________\n",
      "Accuracy[0.8213] Recall[0.7988] F1[0.7886] at fold[1]\n",
      "______________________________________________________\n",
      "Accuracy[0.8725] Recall[0.8837] F1[0.8550] at fold[2]\n",
      "______________________________________________________\n",
      "Accuracy[0.8534] Recall[0.8349] F1[0.8289] at fold[3]\n",
      "______________________________________________________\n",
      "Accuracy[0.8340] Recall[0.8406] F1[0.7986] at fold[4]\n",
      "______________________________________________________\n",
      "Accuracy[0.7386] Recall[0.7564] F1[0.6914] at fold[5]\n",
      "______________________________________________________\n",
      "Accuracy[0.9170] Recall[0.9167] F1[0.8667] at fold[6]\n",
      "______________________________________________________\n",
      "Accuracy[0.8410] Recall[0.8375] F1[0.8172] at fold[7]\n",
      "______________________________________________________\n",
      "Accuracy[0.8207] Recall[0.8288] F1[0.7986] at fold[8]\n",
      "______________________________________________________\n",
      "Accuracy[0.8548] Recall[0.8596] F1[0.8265] at fold[9]\n",
      "______________________________________________________\n"
     ]
    }
   ],
   "source": [
    "avg_acc = []\n",
    "avg_recall = []\n",
    "avg_f1 = []\n",
    "avg_ttime=[]\n",
    "avg_ptime=[]\n",
    "avg_size=[]\n",
    "for i in range(0, len(folds)):\n",
    "    train_idx = folds[i][0]\n",
    "    test_idx = folds[i][1]\n",
    "\n",
    "    X_train, y_train = X[train_idx], y[train_idx]\n",
    "    X_test, y_test = X[test_idx], y[test_idx]\n",
    "\n",
    "    X_train = feature_extraction(X_train)\n",
    "    X_test = feature_extraction(X_test)      \n",
    "        \n",
    "    method = FriedmanTree(max_depth=10,min_samples_split=1,min_samples_leaf=2)\n",
    "    t0=time.time()\n",
    "    method.fit(X_train, y_train)\n",
    "    avg_ttime.append(time.time()-t0)\n",
    "            #Your testing goes here. For instance:\n",
    "    t1=time.time()\n",
    "    y_pred = method.predict(X_test)\n",
    "    avg_ptime.append(time.time()-t1)\n",
    "    y_pred=np.round(y_pred,0)\n",
    "    y_pred=y_pred.astype(int)\n",
    "    v=method.dump()\n",
    "    avg_size.append(round(v.__sizeof__()/1024,3))\n",
    "\n",
    "    acc_fold = accuracy_score(y_test, y_pred)\n",
    "    avg_acc.append(acc_fold)\n",
    "\n",
    "    recall_fold = recall_score(y_test, y_pred, average='macro')\n",
    "    avg_recall.append(recall_fold)\n",
    "\n",
    "    f1_fold  = f1_score(y_test, y_pred, average='macro')\n",
    "    avg_f1.append(f1_fold)\n",
    "\n",
    "    print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.format(acc_fold, recall_fold, f1_fold ,i))\n",
    "    print('______________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Accuracy[0.8553] IC [0.8159, 0.8948]\n",
      "Mean Recall[0.8557] IC [0.8170, 0.8944]\n",
      "Mean F1[0.8272] IC [0.7823, 0.8720]\n",
      "Mean size[1.109]\n",
      "Mean training time[1481.100]\n",
      "Mean prediction time[0.107]\n"
     ]
    }
   ],
   "source": [
    "ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc))\n",
    "ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall))\n",
    "ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1))\n",
    "print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_acc), ic_acc[0], ic_acc[1]))\n",
    "print('Mean Recall[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_recall), ic_recall[0], ic_recall[1]))\n",
    "print('Mean F1[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_f1), ic_f1[0], ic_f1[1]))\n",
    "print('Mean size[{:.3f}]'.format(np.mean(avg_size)))\n",
    "print('Mean training time[{:.3f}]'.format(round(np.mean(avg_ttime)*1000,3)))\n",
    "print('Mean prediction time[{:.3f}]'.format(round(np.mean(avg_ptime)*1000,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. PaloBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy[0.9843] Recall[0.9855] F1[0.9784] at fold[0]\n",
      "______________________________________________________\n",
      "Accuracy[0.7757] Recall[0.7557] F1[0.7250] at fold[1]\n",
      "______________________________________________________\n",
      "Accuracy[0.9243] Recall[0.9294] F1[0.9237] at fold[2]\n",
      "______________________________________________________\n",
      "Accuracy[0.8534] Recall[0.8262] F1[0.8366] at fold[3]\n",
      "______________________________________________________\n",
      "Accuracy[0.9170] Recall[0.9203] F1[0.9122] at fold[4]\n",
      "______________________________________________________\n",
      "Accuracy[0.9432] Recall[0.9291] F1[0.9334] at fold[5]\n",
      "______________________________________________________\n",
      "Accuracy[0.8933] Recall[0.8949] F1[0.8462] at fold[6]\n",
      "______________________________________________________\n",
      "Accuracy[0.7113] Recall[0.7321] F1[0.6540] at fold[7]\n",
      "______________________________________________________\n",
      "Accuracy[0.8008] Recall[0.7478] F1[0.7461] at fold[8]\n",
      "______________________________________________________\n",
      "Accuracy[0.7500] Recall[0.7624] F1[0.6821] at fold[9]\n",
      "______________________________________________________\n"
     ]
    }
   ],
   "source": [
    "avg_acc = []\n",
    "avg_recall = []\n",
    "avg_f1 = []\n",
    "avg_ttime=[]\n",
    "avg_ptime=[]\n",
    "avg_size=[]\n",
    "for i in range(0, len(folds)):\n",
    "    train_idx = folds[i][0]\n",
    "    test_idx = folds[i][1]\n",
    "\n",
    "    X_train, y_train = X[train_idx], y[train_idx]\n",
    "    X_test, y_test = X[test_idx], y[test_idx]\n",
    "\n",
    "    X_train = feature_extraction(X_train)\n",
    "    X_test = feature_extraction(X_test)      \n",
    "        \n",
    "    method = PaloBoost(n_estimators=50,max_depth=6,min_samples_split=1,min_samples_leaf=2)\n",
    "    t0=time.time()\n",
    "    method.fit(X_train, y_train)\n",
    "    avg_ttime.append(time.time()-t0)\n",
    "            #Your testing goes here. For instance:\n",
    "    t1=time.time()\n",
    "    y_pred = method.predict(X_test)\n",
    "    avg_ptime.append(time.time()-t1)\n",
    "    y_pred=np.round(y_pred,0)\n",
    "    y_pred=y_pred.astype(int)\n",
    "    v=method.dump()\n",
    "    avg_size.append(round(v.__sizeof__()/1024,3))\n",
    "\n",
    "    acc_fold = accuracy_score(y_test, y_pred)\n",
    "    avg_acc.append(acc_fold)\n",
    "\n",
    "    recall_fold = recall_score(y_test, y_pred, average='macro')\n",
    "    avg_recall.append(recall_fold)\n",
    "\n",
    "    f1_fold  = f1_score(y_test, y_pred, average='macro')\n",
    "    avg_f1.append(f1_fold)\n",
    "\n",
    "    print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.format(acc_fold, recall_fold, f1_fold ,i))\n",
    "    print('______________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Accuracy[0.8553] IC [0.8022, 0.9084]\n",
      "Mean Recall[0.8483] IC [0.7939, 0.9028]\n",
      "Mean F1[0.8238] IC [0.7571, 0.8904]\n",
      "Mean size[0.211]\n",
      "Mean training time[20185.270]\n",
      "Mean prediction time[4.369]\n"
     ]
    }
   ],
   "source": [
    "ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc))\n",
    "ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall))\n",
    "ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1))\n",
    "print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_acc), ic_acc[0], ic_acc[1]))\n",
    "print('Mean Recall[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_recall), ic_recall[0], ic_recall[1]))\n",
    "print('Mean F1[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_f1), ic_f1[0], ic_f1[1]))\n",
    "print('Mean size[{:.3f}]'.format(np.mean(avg_size)))\n",
    "print('Mean training time[{:.3f}]'.format(round(np.mean(avg_ttime)*1000,3)))\n",
    "print('Mean prediction time[{:.3f}]'.format(round(np.mean(avg_ptime)*1000,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy[0.9922] Recall[0.9928] F1[0.9924] at fold[0]\n",
      "______________________________________________________\n",
      "Accuracy[0.8251] Recall[0.8276] F1[0.8040] at fold[1]\n",
      "______________________________________________________\n",
      "Accuracy[0.9363] Recall[0.9403] F1[0.9308] at fold[2]\n",
      "______________________________________________________\n",
      "Accuracy[0.8872] Recall[0.8586] F1[0.8616] at fold[3]\n",
      "______________________________________________________\n",
      "Accuracy[0.8189] Recall[0.8261] F1[0.7874] at fold[4]\n",
      "______________________________________________________\n",
      "Accuracy[0.9242] Recall[0.9275] F1[0.9135] at fold[5]\n",
      "______________________________________________________\n",
      "Accuracy[0.9170] Recall[0.9167] F1[0.8860] at fold[6]\n",
      "______________________________________________________\n",
      "Accuracy[0.7448] Recall[0.7649] F1[0.7078] at fold[7]\n",
      "______________________________________________________\n",
      "Accuracy[0.7211] Recall[0.6742] F1[0.6438] at fold[8]\n",
      "______________________________________________________\n",
      "Accuracy[0.9113] Recall[0.9088] F1[0.8989] at fold[9]\n",
      "______________________________________________________\n"
     ]
    }
   ],
   "source": [
    "avg_acc = []\n",
    "avg_recall = []\n",
    "avg_f1 = []\n",
    "avg_ttime=[]\n",
    "avg_ptime=[]\n",
    "avg_size=[]\n",
    "for i in range(0, len(folds)):\n",
    "    train_idx = folds[i][0]\n",
    "    test_idx = folds[i][1]\n",
    "\n",
    "    X_train, y_train = X[train_idx], y[train_idx]\n",
    "    X_test, y_test = X[test_idx], y[test_idx]\n",
    "\n",
    "    X_train = feature_extraction(X_train)\n",
    "    X_test = feature_extraction(X_test)      \n",
    "        \n",
    "    method = GBM(n_estimators=50,max_depth=8)\n",
    "    t0=time.time()\n",
    "    method.fit(X_train, y_train)\n",
    "    avg_ttime.append(time.time()-t0)\n",
    "            #Your testing goes here. For instance:\n",
    "    t1=time.time()\n",
    "    y_pred = method.predict(X_test)\n",
    "    avg_ptime.append(time.time()-t1)\n",
    "    y_pred=np.round(y_pred,0)\n",
    "    y_pred=y_pred.astype(int)\n",
    "    v=method.dump()\n",
    "    avg_size.append(round(v.__sizeof__()/1024,3))\n",
    "    acc_fold = accuracy_score(y_test, y_pred)\n",
    "    avg_acc.append(acc_fold)\n",
    "\n",
    "    recall_fold = recall_score(y_test, y_pred, average='macro')\n",
    "    avg_recall.append(recall_fold)\n",
    "\n",
    "    f1_fold  = f1_score(y_test, y_pred, average='macro')\n",
    "    avg_f1.append(f1_fold)\n",
    "\n",
    "    print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.format(acc_fold, recall_fold, f1_fold ,i))\n",
    "    print('______________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Accuracy[0.8678] IC [0.8171, 0.9185]\n",
      "Mean Recall[0.8638] IC [0.8092, 0.9184]\n",
      "Mean F1[0.8426] IC [0.7806, 0.9046]\n",
      "Mean size[0.211]\n",
      "Mean training time[41260.655]\n",
      "Mean prediction time[6.397]\n"
     ]
    }
   ],
   "source": [
    "ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc))\n",
    "ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall))\n",
    "ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1))\n",
    "print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_acc), ic_acc[0], ic_acc[1]))\n",
    "print('Mean Recall[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_recall), ic_recall[0], ic_recall[1]))\n",
    "print('Mean F1[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_f1), ic_f1[0], ic_f1[1]))\n",
    "print('Mean size[{:.3f}]'.format(np.mean(avg_size)))\n",
    "print('Mean training time[{:.3f}]'.format(round(np.mean(avg_ttime)*1000,3)))\n",
    "print('Mean prediction time[{:.3f}]'.format(round(np.mean(avg_ptime)*1000,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Conclusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+---------------+-------------+---------+\n",
      "|     Model     | Mean Accuracy | Mean Recall | Mean F1 |\n",
      "+---------------+---------------+-------------+---------+\n",
      "|    Reg Tree   |     0.8716    |    0.8679   |  0.8431 |\n",
      "|    XGB Tree   |     0.8888    |    0.8944   |  0.8743 |\n",
      "| Friedman Tree |     0.8553    |    0.8532   |  0.8229 |\n",
      "|   Palo Boost  |     0.8477    |    0.8327   |  0.8085 |\n",
      "|      GBM      |     0.8678    |    0.8638   |  0.8426 |\n",
      "+---------------+---------------+-------------+---------+\n",
      "+-------+---------------+-------------+---------+\n",
      "| Model | Mean Accuracy | Mean Recall | Mean F1 |\n",
      "+-------+---------------+-------------+---------+\n",
      "|  Mean |     86.62     |      0      |    0    |\n",
      "+-------+---------------+-------------+---------+\n"
     ]
    }
   ],
   "source": [
    "from prettytable import PrettyTable    \n",
    "x = PrettyTable()\n",
    "x.field_names = [\"Model\", \"Mean Accuracy\", \"Mean Recall\", \"Mean F1\"]\n",
    "x.add_row([\"Reg Tree\", 0.8716,0.8679,0.8431])\n",
    "x.add_row([\"XGB Tree\", 0.8888, 0.8944, 0.8743])\n",
    "x.add_row([\"Friedman Tree\", 0.8553, 0.8532, 0.8229])\n",
    "x.add_row([\"Palo Boost\", 0.8477, 0.8327, 0.8085])\n",
    "x.add_row([\"GBM\", 0.8678, 0.8638, 0.8426])\n",
    "\n",
    "y = PrettyTable()\n",
    "y.field_names = [\"Model\", \"Mean Accuracy\", \"Mean Recall\", \"Mean F1\"]\n",
    "y.add_row([\"Mean\", 86.62, 0, 0])\n",
    "print(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-------+-------+-------+-------+\n",
      "|   Model   |  SNOW |  FNOW |  LOTO |  LOSO |\n",
      "+-----------+-------+-------+-------+-------+\n",
      "| Old paper | 90.03 | 87.78 | 83.41 | 84.52 |\n",
      "| New paper | 98.56 | 97.65 | 85.92 | 86.62 |\n",
      "+-----------+-------+-------+-------+-------+\n"
     ]
    }
   ],
   "source": [
    "y = PrettyTable()\n",
    "y.field_names = [\"Model\", \"SNOW\", \"FNOW\",\"LOTO\", \"LOSO\"]\n",
    "y.add_row([\"Old paper\", 90.03, 87.78, 83.41,84.52])\n",
    "y.add_row([\"New paper\", 98.56, 97.65, 85.92,86.62])\n",
    "\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl4AAAGrCAYAAAARlpmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfbhVZZ3/8fc3UPEB05TUwAGbUdMkUBEI0TKsMXV8SP2hY6GNE1ONVtNPHJ1xGqbRuYwsnWoeMp2kyZT0B4pOpY1FpuLDQcFHHFNRGUmPqAgpCvj9/bHXoS0e4ICsex8O79d1nWvvtda91vpuXdc5H+773mtFZiJJkqT6vaPVBUiSJG0qDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkioRcXJE3NTqOjpExJYRcX1ELIqIq1tdj6S3z+AlaYOLiD+NiLaIWBIRCyLipxExutV1rU1mXpGZH2t1HU2OB3YCdsjME1pdjKS3z+AlaYOKiC8DFwP/RCM0/AHwr8DRraxrbSKid6tr6MRA4H8yc3mrC5G0YRi8JG0wEfFO4KvAX2bm1Mz8XWYuy8zrM3NC1WaLiLg4Ip6pfi6OiC2qbR+OiPkRcVZEPFf1lh0TEYdHxP9ExAsR8TdN55sYEddExJSIWBwR90TEkKbtZ0fEY9W2hyLi2KZtp0bEbRFxUUS8AEys1t1abY9q23PVUN99EbFPx+eMiB9ERHtEPBkR50bEO5qOe2tEXBgRL0bEExHx8TX8N9srImZExEsR8WBEHFWt/wfgK8DYqufwtA32P0pSyxi8JG1IHwT6ANPW0OZvgZHAUGAIMBw4t2n7ztUx+tMIHt8DPgnsDxwEfCUi3tvU/mjgauBdwI+AayNis2rbY9U+7wT+AfhhROzStO8I4HHg3cD5q9T5MeBgYA9gO2AssLDa9u3qmO8FPgSMAz69ynEfAXYEJgGXRUSs+h+iqvN64KaqhjOAKyJiz8z8exq9hlMyc5vMvGzV/SVtfAxekjakHYDn1zI0djLw1cx8LjPbaQSiTzVtXwacn5nLgKtohJd/zszFmfkg8CDwgab2szLzmqr9N2mEtpEAmXl1Zj6TmW9k5hTgURpBr8MzmfntzFyema+uUucyoC/wPiAy8+HMXBARvWiEsHOqmuYB31jlMzyZmd/LzBXAZGAXGsOuqxoJbANckJmvZ+YvgBuAk9bw30/SRszgJWlDWgjsuJb5Uu8BnmxafrJat/IYVWAB6AhDzzZtf5VGWOnwdMebzHwDmN9xvIgYFxGzq2G8l4B9aAS5t+y7qioEfQf4F+DZiLgkIrat9t+8k8/Qv2n5t03HeaV621xzh/cAT1d1r+5YknoQg5ekDWkmsBQ4Zg1tnqExabzDH1Tr1teuHW+qeVYDgGciYiCNYcrTaXwrcDvgAaB5yC/XdODM/FZm7g+8n8aQ4wTgeRq9Yat+hv9dj9qfAXbtmB/2No8laSNg8JK0wWTmIhrzsv6lmhS/VURsFhEfj4hJVbMrgXMjol9E7Fi1/+HbOO3+EfGJqpftS8BrwB3A1jSCVTtARHyaRo9Xl0TEARExopqH9TsagXJF1Rv3Y+D8iOhbBbwvr+dnuLM69lnVf6cPA39CY4hVUg9k8JK0QWXmN2kEkXNphJ6nafQ6XVs1OQ9oA+4D7gfuqdatr+tozLl6kcY8q09U36R8iMbcq5k0hioHA7etw3G3pdFj9iKN4b+FwIXVtjNoBKbHgVtpTOr/j3UtPDNfB44CPk6jJ+1fgXGZOXddjyVp4xCZa+xpl6RuKyImAn+UmZ9sdS2S1BX2eEmSJBVi8JIkSSrEoUZJkqRC7PGSJEkqpDs+FPYtdtxxxxw0aFCry5AkSVqrWbNmPZ+Z/TrbtlEEr0GDBtHW1tbqMiRJktYqIp5c3TaHGiVJkgoxeEmSJBVi8JIkSSpko5jjJUnShrJs2TLmz5/P0qVLW12KNnJ9+vRhwIABbLbZZl3ex+AlSdqkzJ8/n759+zJo0CAiotXlaCOVmSxcuJD58+ez2267dXk/hxolSZuUpUuXssMOOxi69LZEBDvssMM695wavCRJmxxDlzaE9bmODF6SJEmFOMdLkrRJG3T2f23Q48274Ii1ttlmm21YsmTJeh1/4sSJbLPNNpx55pnrtf+afPGLX+Saa67h6aef5h3v6LxvZvbs2TzzzDMcfvjhG/z8q7r22mvZY4892Hvvvbu8z+WXX05bWxvf+c53VttmxowZHH300Rx44IH85Cc/AeCwww7jjjvuYPTo0dxwww0r25588sn89Kc/5ZJLLuH4449f/w9TscdLkqRuasWKFcXO9cYbbzBt2jR23XVXbrnllk7bLF++nNmzZ68MK12VmbzxxhvrXNO1117LQw89tM77dcVBBx30ps8xYcIE/vM///Mt7a644gqOOuqoDXZeg5ckSS2QmUyYMIF99tmHwYMHM2XKFKDRG3PIIYfwp3/6pwwePBiA888/nz333JNDDz2URx55ZOUxvve973HAAQcwZMgQjjvuOF555RUAnnzyScaMGcMHPvABxowZw1NPPbXWen75y1+yzz778LnPfY4rr7xy5fqJEycyfvx4PvaxjzFu3Di+8pWvMGXKFIYOHcqUKVOYOHEiF1544cr2++yzD/PmzWPevHnstddefP7zn2e//fbj6aefXu25H3vsMQ477DD2339/DjroIObOncvtt9/O9OnTmTBhAkOHDuWxxx570z7XX389I0aMYN999+XQQw/l2Wef7cJ/9dUbM2YMffv2fVvH6AqDlyRJLTB16lRmz57NnDlz+O///m8mTJjAggULALjrrrs4//zzeeihh5g1axZXXXUV9957L1OnTuXuu+9eeYxPfOIT3H333cyZM4e99tqLyy67DIDTTz+dcePGcd9993HyySfzhS98Ya31XHnllZx00kkce+yx3HDDDSxbtmzltlmzZnHdddfxox/9iK9+9auMHTuW2bNnM3bs2DUe85FHHmHcuHHce++9DBw4cLXtxo8fz7e//W1mzZrFhRdeyOc//3lGjRrFUUcdxde//nVmz57NH/7hH75pn9GjR3PHHXdw7733cuKJJzJp0qS1fsbuwDlekiS1wK233spJJ51Er1692GmnnfjQhz7E3Xffzbbbbsvw4cNX3hvq17/+NcceeyxbbbUVwJuGvR544AHOPfdcXnrpJZYsWcIf//EfAzBz5kymTp0KwKc+9SnOOuusNdby+uuv85Of/ISLLrqIvn37MmLECG666SaOOOKIlefccsst1/kzDhw4kJEjR66xzZIlS7j99ts54YQTVq577bXX1nrs+fPnM3bsWBYsWMDrr7++TvfSaiWDlyRJLZCZq9229dZbv2l5dbctOPXUU7n22msZMmQIl19+OTNmzOi03dpue/Czn/2MRYsWrRzafOWVV9hqq61WBq9V62nWu3fvN83far6v1Zr26/DGG2+w3XbbMXv27LW2bXbGGWfw5S9/maOOOooZM2YwceLEddq/VRxqlCSpBQ4++GCmTJnCihUraG9v55ZbbmH48OGdtps2bRqvvvoqixcv5vrrr1+5bfHixeyyyy4sW7aMK664YuX6UaNGcdVVVwGNyeGjR48GYNq0aZxzzjlvOceVV17JpZdeunJu1hNPPMFNN920cs5Ys759+7J48eKVy4MGDeKee+4B4J577uGJJ55Y7WceN24cd91115vWbbvttuy2225cffXVQCOQzpkzp9NzNVu0aBH9+/cHYPLkyas9Z3djj5ckaZPWlds/bEjLly9niy224Nhjj2XmzJkMGTKEiGDSpEnsvPPOzJ07903t99tvP8aOHcvQoUMZOHAgBx100Mpt//iP/8iIESMYOHAggwcPXhlSvvWtb/Fnf/ZnfP3rX6dfv358//vfBxqT2Lfddts3Hf+VV17hxhtv5Lvf/e7KdVtvvTWjR49+U8jrcMghh3DBBRcwdOhQzjnnHI477jh+8IMfMHToUA444AD22GOP1X72++67j1122eUt66+44go+97nPcd5557Fs2TJOPPFEhgwZwoknnshnPvMZvvWtb3HNNde8aZ7XxIkTOeGEE+jfvz8jR47sNPBNnz6dtrY2vvrVr662pg4dk/qXLFnCgAEDuOyyy1YO3W5Isaauzu5i2LBh2dbWVu9JJr6z3uO3wsRFra5Akrqdhx9+mL322qtl558zZw6f+cxn3tLzU8InP/lJLrroIvr161f83C+//DKnnXbayp6tVpoxYwYXXnjhm+7XtSannnoqRx55ZKf38erseoqIWZk5rLNjOdQoSVIh//7v/85JJ53Eeeed15Lz//CHP2xJ6ILGkGJ3CF0Am2++OQ888ECXbgJ78skn86tf/Yo+ffpskHM71ChJUiGf/exn+exnP9vqMjZ5o0aNYt68eV1q2zx3bkOwx0uSJKkQg5ckSVIhBi9JkqRCnOMlleC3ZiVJGLwkSZu6Df0Poy78o6RXr14MHjyYzKRXr1585zvfYdSoURu2jk5Mnz6dhx56iLPPPrv2c6lzBi9JkgrbcsstVz4i58Ybb+Scc87hV7/6Ve3nPeqoo970rEeV5xwvSZJa6OWXX2b77bcHGo/LmTBhAvvssw+DBw9mypQpQOOGnx/+8Ic5/vjjed/73sfJJ5+88lmPZ599NnvvvTcf+MAHOPPMMwG4/vrrGTFiBPvuuy+HHnoozz77LACXX345p59+egs+pTrY4yVJUmGvvvoqQ4cOZenSpSxYsIBf/OIXAEydOpXZs2czZ84cnn/+eQ444AAOPvhgAO69914efPBB3vOe93DggQdy2223sffeezNt2jTmzp1LRPDSSy8BMHr0aO644w4igksvvZRJkybxjW98o2WfV79n8JIkqbDmocaZM2cybtw4HnjgAW699VZOOukkevXqxU477cSHPvQh7r77brbddluGDx/OgAEDABg6dCjz5s1j5MiR9OnThz//8z/niCOO4MgjjwRg/vz5jB07lgULFvD666+z2267teyz6s0capQkqYU++MEP8vzzz9Pe3s6anp+8xRZbrHzfq1cvli9fTu/evbnrrrs47rjjuPbaaznssMMAOOOMMzj99NO5//77+e53v8vSpUtr/xzqGoOXJEktNHfuXFasWMEOO+zAwQcfzJQpU1ixYgXt7e3ccsstDB8+fLX7LlmyhEWLFnH44Ydz8cUXr+xFW7RoEf379wdg8uTJRT6HusahRknSpq0F96TrmOMFjQn1kydPplevXhx77LHMnDmTIUOGEBFMmjSJnXfemblz53Z6nMWLF3P00UezdOlSMpOLLroIgIkTJ3LCCSfQv39/Ro4cyRNPPFHss2nNYk3dmt3FsGHDsq2trd6TeINL1cnrS+o2Hn74Yfbaa69Wl6EeorPrKSJmZeawzto71ChJklSIwUuSJKkQg5ckaZOzMUyzUfe3PteRwUuStEnp06cPCxcuNHzpbclMFi5cSJ8+fdZpP7/VKEnapAwYMID58+fT3t7e6lK0kevTp8/Km9p2lcFLkrRJ2WyzzbyTu1rGoUZJkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhRi8JEmSCjF4SZIkFeJDsiVJ0ltNfGerK9jwJi5qdQX2eEmSJJVi8JIkSSrEoUZJ2pg5HCRtVOzxkiRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhRi8JEmSCjF4SZIkFWLwkiRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhRi8JEmSCjF4SZIkFWLwkiRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhRi8JEmSCjF4SZIkFWLwkiRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhRi8JEmSCjF4SZIkFWLwkiRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUyFqDV0ScHhHblyhGkiSpJ+tKj9fOwN0R8eOIOCwiou6iJEmSeqK1Bq/MPBfYHbgMOBV4NCL+KSL+sObaJEmSepQuzfHKzAR+W/0sB7YHromISTXWJkmS1KP0XluDiPgCcArwPHApMCEzl0XEO4BHgbPqLVGSJKlnWGvwAnYEPpGZTzavzMw3IuLIesqSJEnqeboy1PgT4IWOhYjoGxEjADLz4boKkyRJ6mm6Erz+DVjStPy7ap0kSZLWQVeCV1ST64HGECNdG6KUJElSk64Er8cj4gsRsVn180Xg8boLkyRJ6mm6Erw+C4wC/heYD4wAxtdZlCRJUk+01iHDzHwOOLFALZIkST1aV+7j1Qc4DXg/0KdjfWb+WY11SZIk9ThdGWr8TxrPa/xj4FfAAGBxnUVJkiT1RF0JXn+UmX8H/C4zJwNHAIPrLUuSJKnn6UrwWla9vhQR+wDvBAbVVpEkSVIP1ZX7cV0SEdsD5wLTgW2Av6u1KkmSpB5ojcGrehD2y5n5InAL8N4iVUmSJPVAaxxqrO5Sf3qhWiRJknq0rszx+nlEnBkRu0bEuzp+aq9MkiSph+nKHK+O+3X9ZdO6xGFHSZKkddKVO9fvVqIQSZKknq4rd64f19n6zPzBhi9HkiSp5+rKUOMBTe/7AGOAewCDlyRJ0jroylDjGc3LEfFOGo8RkiRJ0jroyrcaV/UKsPuGLkSSJKmn68ocr+tpfIsRGkFtb+DHdRYlSZLUE3VljteFTe+XA09m5vya6pEkSeqxuhK8ngIWZOZSgIjYMiIGZea8WiuTJEnqYboyx+tq4I2m5RXVOkmSJK2DrgSv3pn5esdC9X7z+kqSJEnqmboSvNoj4qiOhYg4Gni+vpIkSZJ6pq7M8foscEVEfKdang90ejd7SZIkrV5XbqD6GDAyIrYBIjMX11+WJElSz7PWocaI+KeI2C4zl2Tm4ojYPiLOK1GcJElST9KVOV4fz8yXOhYy80Xg8PpKkiRJ6pm6Erx6RcQWHQsRsSWwxRraS5IkqRNdmVz/Q+DmiPh+tfxpYHJ9JUmSJPVMXZlcPyki7gMOBQL4GTCw7sJKG7T0R60uYYOb1+oCJEnSm3RlqBHgtzTuXn8cMAZ4uLaKJEmSeqjV9nhFxB7AicBJwEJgCo3bSRxSqDZJkqQeZU1DjXOBXwN/kpm/AYiIvypSlSRJUg+0puB1HI0er19GxM+Aq2jM8ZK0jpxDKEmCNczxysxpmTkWeB8wA/grYKeI+LeI+Fih+iRJknqMtU6uz8zfZeYVmXkkMACYDZxde2WSJEk9TFe/1QhAZr6Qmd/NzI/UVZAkSVJPtU7BS5IkSevP4CVJklRIVx4ZJEmSNjF+G7se9nhJkiQVYvCSJEkqxKFGSdqIORwkbVzs8ZIkSSrE4CVJklRIrcErIv4qIh6MiAci4sqI6BMRu0XEnRHxaERMiYjN66xBkiSpu6gteEVEf+ALwLDM3AfoReOh218DLsrM3YEXgdPqqkGSJKk7qXuosTewZUT0BrYCFgAfAa6ptk8Gjqm5BkmSpG6htuCVmf8LXAg8RSNwLQJmAS9l5vKq2Xygf2f7R8T4iGiLiLb29va6ypQkSSqmzqHG7YGjgd2A9wBbAx/vpGl2tn9mXpKZwzJzWL9+/eoqU5IkqZg6hxoPBZ7IzPbMXAZMBUYB21VDjwADgGdqrEGSJKnbqDN4PQWMjIitIiKAMcBDwC+B46s2pwDX1ViDJElSt1HnHK87aUyivwe4vzrXJcBfA1+OiN8AOwCX1VWDJElSd1LrI4My8++Bv19l9ePA8DrPK0mS1B1553pJkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhdQWvCJiz4iY3fTzckR8KSLeFRE/j4hHq9ft66pBkiSpO6nzIdmPZObQzBwK7A+8AkwDzgZuzszdgZurZUmSpB6v1FDjGOCxzHwSOBqYXK2fDBxTqAZJkqSWKhW8TgSurN7vlJkLAKrXd3e2Q0SMj4i2iGhrb28vVKYkSVJ9ag9eEbE5cBRw9brsl5mXZOawzBzWr1+/eoqTJEkqqESP18eBezLz2Wr52YjYBaB6fa5ADZIkSS1XInidxO+HGQGmA6dU708BritQgyRJUsvVGrwiYivgo8DUptUXAB+NiEerbRfUWYMkSVJ30bvOg2fmK8AOq6xbSONbjpIkSZsU71wvSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhRi8JEmSCjF4SZIkFWLwkiRJKsTgJUmSVIjBS5IkqZC6n9W4XURcExFzI+LhiPhgRLwrIn4eEY9Wr9vXWYMkSVJ3UXeP1z8DP8vM9wFDgIeBs4GbM3N34OZqWZIkqcerLXhFxLbAwcBlAJn5ema+BBwNTK6aTQaOqasGSZKk7qTOHq/3Au3A9yPi3oi4NCK2BnbKzAUA1eu7a6xBkiSp26gzePUG9gP+LTP3BX7HOgwrRsT4iGiLiLb29va6apQkSSqmzuA1H5ifmXdWy9fQCGLPRsQuANXrc53tnJmXZOawzBzWr1+/GsuUJEkqo7bglZm/BZ6OiD2rVWOAh4DpwCnVulOA6+qqQZIkqTvpXfPxzwCuiIjNgceBT9MIez+OiNOAp4ATaq5BkiSpW6g1eGXmbGBYJ5vG1HleSZKk7sg710uSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUSK3PaoyIecBiYAWwPDOHRcS7gCnAIGAe8H8y88U665AkSeoOSvR4HZKZQzOz42HZZwM3Z+buwM3VsiRJUo/XiqHGo4HJ1fvJwDEtqEGSJKm4uoNXAjdFxKyIGF+t2ykzFwBUr+/ubMeIGB8RbRHR1t7eXnOZkiRJ9at1jhdwYGY+ExHvBn4eEXO7umNmXgJcAjBs2LCsq0BJkqRSau3xysxnqtfngGnAcODZiNgFoHp9rs4aJEmSuovagldEbB0RfTveAx8DHgCmA6dUzU4BrqurBkmSpO6kzqHGnYBpEdFxnh9l5s8i4m7gxxFxGvAUcEKNNUiSJHUbtQWvzHwcGNLJ+oXAmLrOK0mS1F1553pJkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQgxekiRJhdQevCKiV0TcGxE3VMu7RcSdEfFoREyJiM3rrkGSJKk7KNHj9UXg4ablrwEXZebuwIvAaQVqkCRJarlag1dEDACOAC6tlgP4CHBN1WQycEydNUiSJHUXdfd4XQycBbxRLe8AvJSZy6vl+UD/znaMiPER0RYRbe3t7TWXKUmSVL/agldEHAk8l5mzmld30jQ72z8zL8nMYZk5rF+/frXUKEmSVFLvGo99IHBURBwO9AG2pdEDtl1E9K56vQYAz9RYgyRJUrdRW49XZp6TmQMycxBwIvCLzDwZ+CVwfNXsFOC6umqQJEnqTlpxH6+/Br4cEb+hMefrshbUIEmSVFydQ40rZeYMYEb1/nFgeInzSpIkdSfeuV6SJKkQg5ckSVIhBi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIbUFr4joExF3RcSciHgwIv6hWr9bRNwZEY9GxJSI2LyuGiRJkrqTOnu8XgM+kplDgKHAYRExEvgacFFm7g68CJxWYw2SJEndRm3BKxuWVIubVT8JfAS4plo/GTimrhokSZK6k1rneEVEr4iYDTwH/Bx4DHgpM5dXTeYD/Vez7/iIaIuItvb29jrLlCRJKqLW4JWZKzJzKDAAGA7s1Vmz1ex7SWYOy8xh/fr1q7NMSZKkIop8qzEzXwJmACOB7SKid7VpAPBMiRokSZJarc5vNfaLiO2q91sChwIPA78Ejq+anQJcV1cNkiRJ3UnvtTdZb7sAkyOiF42A9+PMvCEiHgKuiojzgHuBy2qsQZIkqduoLXhl5n3Avp2sf5zGfC9JkqRNineulyRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklRInQ/J3jUifhkRD0fEgxHxxWr9uyLi5xHxaPW6fV01SJIkdSd19ngtB/5vZu4FjAT+MiL2Bs4Gbs7M3YGbq2VJkqQer7bglZkLMvOe6v1i4GGgP3A0MLlqNhk4pq4aJEmSupMic7wiYhCwL3AnsFNmLoBGOAPevZp9xkdEW0S0tbe3lyhTkiSpVrUHr4jYBvh/wJcy8+Wu7peZl2TmsMwc1q9fv/oKlCRJKqTW4BURm9EIXVdk5tRq9bMRsUu1fRfguTprkCRJ6i7q/FZjAJcBD2fmN5s2TQdOqd6fAvevYOAAAAd7SURBVFxXVw2SJEndSe8aj30g8Cng/oiYXa37G+AC4McRcRrwFHBCjTVIkiR1G7UFr8y8FYjVbB5T13klSZK6K+9cL0mSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgoxeEmSJBVi8JIkSSrE4CVJklSIwUuSJKkQg5ckSVIhtQWviPiPiHguIh5oWveuiPh5RDxavW5f1/klSZK6mzp7vC4HDltl3dnAzZm5O3BztSxJkrRJqC14ZeYtwAurrD4amFy9nwwcU9f5JUmSupvSc7x2yswFANXru1fXMCLGR0RbRLS1t7cXK1CSJKku3XZyfWZekpnDMnNYv379Wl2OJEnS21Y6eD0bEbsAVK/PFT6/JElSy5QOXtOBU6r3pwDXFT6/JElSy9R5O4krgZnAnhExPyJOAy4APhoRjwIfrZYlSZI2Cb3rOnBmnrSaTWPqOqckSVJ31m0n10uSJPU0Bi9JkqRCDF6SJEmFGLwkSZIKMXhJkiQVYvCSJEkqxOAlSZJUiMFLkiSpEIOXJElSIQYvSZKkQloSvCLisIh4JCJ+ExFnt6IGSZKk0ooHr4joBfwL8HFgb+CkiNi7dB2SJEmltaLHazjwm8x8PDNfB64Cjm5BHZIkSUVFZpY9YcTxwGGZ+efV8qeAEZl5+irtxgPjq8U9gUeKFlqvHYHnW12EeiSvLdXJ60t16WnX1sDM7NfZht6lKwGik3VvSX+ZeQlwSf3llBcRbZk5rNV1qOfx2lKdvL5Ul03p2mrFUON8YNem5QHAMy2oQ5IkqahWBK+7gd0jYreI2Bw4EZjegjokSZKKKj7UmJnLI+J04EagF/Afmflg6TparEcOoapb8NpSnby+VJdN5toqPrlekiRpU+Wd6yVJkgoxeEmSJBVi8HobIuJvI+LBiLgvImZHxIiImBERbU1thkXEjKbl0RFxV0TMrX7GV+u3i4iFERHV8gcjIiNiQLX8zoh4ISL8f7YJiIgV1TXV8TMoIj5cXRN/0tTuhoj4cPV+84i4OCIei4hHI+K6puvnooj4UtN+N0bEpU3L34iILxf8iGqxiFiymvXjm34/3RURo6v106pr8TcRsajp2hy1pmtPm551vbaqbUdGxL0RMSciHoqIv+jKfhujVtzHq0eIiA8CRwL7ZeZrEbEjsHm1+d0R8fHM/Okq++wM/Ag4JjPvqfa5MSL+NzP/KyJ+C+wFPASMAu6tXn8MjATuzMw3inxAtdqrmTm0eUVEDKJxO5a/Ba7vZJ9/AvoCe2Tmioj4NDA1IkYAtwMnABdX4X1HYNumfUcBX1r1gNq0RMSRwF8AozPz+YjYD7g2IoZn5rFVmw8DZ2bmkU37Xchqrr10IrFY87UFLKQxuX54Zs6PiC2AQWvbLzN/25IP8zbZe7L+dgGez8zXADLz+czsuB/Z14FzO9nnL4HLM/Oejn2As4COB4XfRuMPINXrRass376hP4Q2OnOARRHx0eaVEbEV8GngrzJzBUBmfh94DfgIb7623g88ACyOiO2rX3J70Qj62rT9NTCh+t1E9btqMo3fXZ3qwrUnwZqvrb40OoIWVttey8xHurDfRsngtf5uAnaNiP+JiH+NiA81bZsJvBYRh6yyz/uBWausa6vWQyNYdfxxfC9wNdBxJ99RNP54atOwZdNQzrRVtp3HW4P9HwFPZebLq6xvA95f/aNgeUT8AY1raSZwJ/BBGtfYfdWzU7VpW9vvqM6s8drbgLVp47baayszX6BxP88nI+LKiDi5aVrN+lyT3ZrBaz1l5hJgfxrPk2wHpkTEqU1NOvvjGHTyeKSmdbcBoyJiN2BeZi4FIiK2qc5114b7BOrmXs3ModXPsc0bMvPXABFxUNPq1V1bzes7er06gtfMpmV7U7U6q7u21rZ9bftJK6+R6vnNY2j8nTsT+I+u7LcxMni9DZm5IjNnZObfA6cDxzVt+wXQh8bcrA4P8vserA7705jTRWY+CmwP/AmNP4rQSPqfBp6owp4EcD6NuV4dfgMMjIi+q7Tbj+r64vc9qoNpDDXeQaPHy95UdXiIxu+kZs3XUGe6cu1Ja722MvP+zLwI+Ci//3u6Ptdkt2bwWk8RsWdE7N60aijw5CrNzqcxh6vDvwCnRsTQ6hg7AF8DJjW1mQl8kd8Hr5k0Jj3bI6GVMvMmGiF9SLX8OxrzHr4ZEb0AImIcsBXwi2q322h8IeSF6h8NLwDb0QhfM5Eav4u+Vv1uovpddSrwr6vboYvXnrTaaysitun4dnal+e/pOl+T3Z3falx/2wDfjojtgOU0/tU3Hrimo0Fm/iQi2puWF0TEJ4HvVf86DODizGz+htptwOE0xrCh8QfxvRi89FbnA9c1LZ8DXAj8T0S8AcwFjm36Vtn9NL7N+KOmfe4HtumYuKpNylYRMb9p+ZuZ+c2I6A/cHhEJLAY+mZkL1nKstV172rSs07VV/T08KyK+C7wK/I5GuCIzp6/nNdlt+cggSZKkQhxqlCRJKsTgJUmSVIjBS5IkqRCDlyRJUiEGL0mSpEIMXpIkSYUYvCRJkgr5/1rmvXvmpgtMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np \n",
    "import matplotlib.pyplot as plt \n",
    "  \n",
    "N = 4\n",
    "  \n",
    "boys = (90.03, 87.78, 83.41, 84.52) \n",
    "girls = (98.56, 97.65, 85.92, 86.62) \n",
    "boyStd = (2, 3, 4, 1, 2) \n",
    "girlStd = (3, 5, 2, 3, 3) \n",
    "ind = np.arange(N)    \n",
    "width = 0.35  \n",
    "  \n",
    "fig = plt.subplots(figsize =(10, 7)) \n",
    "p1 = plt.bar(ind, boys, width)#, yerr = boyStd) \n",
    "p2 = plt.bar(ind, girls, width, \n",
    "             bottom = boys)#, yerr = girlStd) \n",
    "  \n",
    "plt.ylabel('Accuracy') \n",
    "plt.title('Comparison of') \n",
    "plt.xticks(ind, ('SNOW', 'FNOW', 'LOTO', 'LOSO')) \n",
    "plt.yticks(np.arange(0, 81, 10)) \n",
    "plt.legend((p1[0], p2[0]), ('Jordao, Artur, et al. [1]', 'Bonsai')) \n",
    "  \n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAEvCAYAAAA+QnQyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAfZUlEQVR4nO3de5QU5Z3/8fc3oCDeUEQlQIA1xo3RA6uIBi/BuEm8sBCjBDyuyJos0YjRNdHg6llHoydETGBNshtRN2AkajSI4M+omyheSRBkRERcL9EsK9mMsiKIXH1+f3QxNjCXBrqnZ6ber3P6TNfTVV3fppj5dD31VFWklJAkqb37WLULkCSpJRh4kqRcMPAkSblg4EmScsHAkyTlgoEnScqFjtUuYGfst99+qW/fvtUuQ5LUiixYsODtlFL3rdvbdOD17duX+fPnV7sMSVIrEhFvNtRul6YkKRcMPElSLhh4kqRcqNgxvIj4D2Ao8JeU0mFZ277A3UBf4A3gqyml/4uIAP4VOBVYA4xJKT1XqdokVc+GDRtYtmwZa9eurXYpauM6d+5Mr1692GWXXUqav5KDVqYCPwFuL2obD/wupTQhIsZn098FTgEOzh5HA/+e/ZTUzixbtow999yTvn37UviuK22/lBLvvPMOy5Yto1+/fiUtU7EuzZTSE8CKrZqHA9Oy59OALxe1354Kfg90jYgelapNUvWsXbuWbt26GXbaKRFBt27dtqunoKWP4R2QUloOkP3cP2vvCfx30XzLsjZJ7ZBhp3LY3v9HrWXQSkNVN3ijvogYGxHzI2J+XV1dhcuS1B7tscceO7xsTU0NN954Yxmr+cjFF19Mz549+fDDDxudp7a2lgcffLAi69/azJkzWbJkyXYtM3XqVMaNG9fkPHPmzGHvvffm1FNPrW87+eST6dq1K0OHDt1i3rPPPpt9992Xe++9d7vqaEhLn3j+vxHRI6W0POuy/EvWvgzoXTRfL+Ctht4gpTQFmAIwcOBA714rtXGTJ0xg5bp1ZXu/vTt14pLx48vyXps2baJDhw5lea/mfPjhh9x333307t2bJ554giFDhmwzz8aNG6mtrWX+/PlbhEVzUkqklPjYx7ZvH2fmzJkMHTqUQw89dLuWK8Xxxx/PAw88UD992WWXsWbNGm6++eYt5ps+fTpjxowpyzpbOvBmAecCE7Kf9xe1j4uIuygMVlm5uetTUvu2ct06rq6pKdv7XVPie6WUuPzyy/nNb35DRHDVVVcxcuRI5syZwzXXXEOPHj2ora1lyZIlXH/99dx+++307t2b7t27c+SRRwJwyy23MGXKFNavX88nP/lJfvGLX9ClSxfefPNNzjvvPOrq6ujevTs///nP+cQnPtFkPY899hiHHXYYI0eO5M4776wPvJqaGt566y3eeOMN9ttvP5566ik++OADnnrqKa644gpeeukl9thjD77zne8AcNhhh9UHySmnnMKJJ57I3LlzmTlzJn369Glw3a+99hoXXnghdXV1dOnShVtuuYUVK1Ywa9YsHn/8ca677jp+/etfc9BBB9UvM3v2bK677jrWr19Pt27dmD59OgcccEBJ//YNOemkk5gzZ84OL1+KinVpRsSdwFzgkIhYFhFfoxB0X4iIV4AvZNMADwKvA68CtwDfrFRdkgQwY8YMamtref755/ntb3/LZZddxvLlhe/Z8+bN4/rrr2fJkiUsWLCAu+66i4ULFzJjxgyeffbZ+vf4yle+wrPPPsvzzz/Ppz/9aW677TYAxo0bx+jRo1m0aBFnn3023/rWt5qt58477+Sss87i9NNP54EHHmDDhg31ry1YsID777+fX/7yl1x77bWMHDmS2tpaRo4c2eR7vvzyy4wePZqFCxc2GnYAY8eO5cc//jELFizgxhtv5Jvf/CaDBw9m2LBhTJw4kdra2i3CDuC4447j97//PQsXLmTUqFHccMMNzX7GaqvYHl5K6axGXjqpgXkTcGGlamkNJkyYzLp1Kyu+nk6d9mb8+Esqvh6prXvqqac466yz6NChAwcccACf+9znePbZZ9lrr70YNGhQ/VD3J598ktNPP50uXboAMGzYsPr3WLx4MVdddRXvvvsuq1ev5ktf+hIAc+fOZcaMGQCcc845XH755U3Wsn79eh588EEmTZrEnnvuydFHH80jjzzCaaedVr/O3Xbbbbs/Y58+fTjmmGOanGf16tU888wzjBgxor5tXQldzMuWLWPkyJEsX76c9evXl3xqQDW16YtHtyXr1q2kpubqiq+npuaaiq9Dag8K37Mbtvvuu28x3dhowDFjxjBz5kz69+/P1KlTG+2Sa2404UMPPcTKlSs5/PDDAVizZg1dunSpD7yt6ynWsWPHLQa5FA/Tb2q5zT788EO6du1KbW1ts/MWu+iii7j00ksZNmwYc+bMoaaM3dKVYuAp11piz9u97tbphBNO4Oabb+bcc89lxYoVPPHEE0ycOJGlS5duM9+YMWMYP348GzduZPbs2XzjG98AYNWqVfTo0YMNGzYwffp0evYsnE01ePBg7rrrLs455xymT5/OcccdB8B9993HvHnz+P73v7/FOu68805uvfVWzjqr0DH2/vvv069fP9asWbNN3XvuuSerVq2qn+7bt2/9MbvnnnuOP/7xj41+5tGjRzNu3DgGDRpU37bXXnvRr18/7rnnHkaMGEFKiUWLFtG/f/9t1lVs5cqV9Z932rRpDc7T2hh4yrWW2PN2r7t12bhxI506deL0009n7ty59O/fn4jghhtu4MADD9wm8I444ghGjhzJgAED6NOnD8cff3z9a9/73vc4+uij6dOnD4cffnh9ONx0002cd955TJw4sX7QChQGh+y1115bvP+aNWt4+OGHtxiduPvuu3Pccccxe/bsbeo/8cQTmTBhAgMGDOCKK67gjDPO4Pbbb2fAgAEcddRRfOpTn2r0sy9atIgePba9psf06dO54IILuO6669iwYQOjRo2if//+jBo1in/8x3/kpptu4t57793iOF5NTQ0jRoygZ8+eHHPMMQ0G7axZs5g/fz7XXnttozVtdvzxx7N06VJWr15Nr169uO222+q7iMvFwJNUVXt36lTyyMpS368pL774IgcddBARwcSJE5k4ceIWrw8ZMmSbUwKuvPJKrrzyym3e64ILLuCCCy7Ypr1v3748+uij27TX1tYyadKkLdq6dOnCihVbX5SK+mOAW9t33323GDgD8MgjjzQ47+LFi+ufv/feexx88MH07t17m/n69evHQw89tE37scce2+h5eMOHD2f48OHbtI8ZM6b+NIJhw4ZtccyzKU8++WRJ8+0MA09SVZXrnLlS/OxnP+Omm25i8uTJLbbOYnfccUdV1guFrst77rmnausvtuuuu7J48WJOPfXUZk+iP/vss3nmmWc488wzd3q9Bp6k3Dj//PM5//zzq11G7g0ePJg33nijpHmnT59etvW2lkuLSZJUUQaeJCkXDDxJUi4YeJKkXDDwJOVOhw4dGDBgAP379+eII47gmWeeaZH1zpo1iwkTJjQ/oyrCUZqSqqrcV7sp5co2u+22W/2ltB5++GGuuOIKHn/88bLV0JjtOS9N5WfgSaqqcl/tZnuvbPPee++xzz77AE3fMqimpob99tuPxYsXc+SRR3LHHXcQEYwfP55Zs2bRsWNHvvjFL3LjjTc2euucqVOnMn/+fH7yk5+U7fOqdAaepNz54IMPGDBgAGvXrmX58uX1V0UpvmXQ22+/zVFHHcUJJ5wAwMKFC3nxxRf5+Mc/zrHHHsvTTz/NoYceyn333cfSpUuJCN59913go1vnRAS33norN9xwAz/84Q+r9nlVYOBJyp3iLs25c+cyevRoFi9e3Owtg3r16gXAgAEDeOONNzjmmGPo3LkzX//61znttNMYOnQo0DZvnZMHDlqRlGuf/exnefvtt6mrq2vylkGdiq7R2aFDBzZu3EjHjh2ZN28eZ5xxBjNnzuTkk08GCrfOGTduHC+88AI333zzFrfsUfUYeJJybenSpWzatIlu3bpxwgkncPfdd7Np0ybq6up44okntriVztZWr17NypUrOfXUU5k8eXL9XmNbvHVOHtilKSl3Nh/Dg8JAlWnTptGhQ4eSbxm02apVqxg+fDhr164lpVR/J4RSbp2jlmfgSaqqTp32Lus9Azt12rvZeTZt2tRge6m3DCoeZTlv3rxt3qeUW+eo5Rl4kqrKu8GrpXgMT5KUC+7hAZMnTGDlunXVLkOSVEEGHrBy3Tqurqmp6DquqfD7S21JSomIqHYZauOaOo2kIXZpSmpRnTt35p133tnuP1ZSsZQS77zzDp07dy55GffwJLWoXr16sWzZMurq6qpditq4zp0711/9phQGnqQWtcsuu3ipLVWFXZqSpFxwD0+tkiNnJZWbgadWqSVGzoKjZ6U8sUtTkpQL7uFJkpo1YcJk1q1bWdF1dOq0d0UvNWfgSZKatW7dSmpqrq7oOsp5EfGGGHiSWpX2sCeh1snAk9SqtIc9CbVODlqRJOWCgSdJygW7NCWpDfMiDaUz8CSpDfMiDaWzS1OSlAvu4Ukqmd1nassMPEkla4nus/bQdabWyS5NSVIuGHiSpFww8CRJuVCVwIuIf4qIFyNicUTcGRGdI6JfRPwhIl6JiLsjYtdq1CZJap9aPPAioifwLWBgSukwoAMwCvgBMCmldDDwf8DXWro2SVL7Va0uzY7AbhHREegCLAc+D9ybvT4N+HKVapMktUMtHngppf8BbgT+RCHoVgILgHdTShuz2ZYBPVu6NklS+1WNLs19gOFAP+DjwO7AKQ3MmhpZfmxEzI+I+XV1dZUrVJLUrlSjS/NvgT+mlOpSShuAGcBgoGvWxQnQC3iroYVTSlNSSgNTSgO7d+/eMhVLktq8agTen4BjIqJLRARwErAEeAw4M5vnXOD+KtQmSWqnqnEM7w8UBqc8B7yQ1TAF+C5waUS8CnQDbmvp2iRJ7VdVrqWZUroauHqr5teBQVUoR5KUA15pRZKUCwaeJCkXDDxJUi4YeJKkXDDwJEm5YOBJknLBwJMk5YKBJ0nKBQNPkpQLBp4kKRcMPElSLhh4kqRcMPAkSblg4EmScsHAkyTlgoEnScoFA0+SlAsGniQpFww8SVIuGHiSpFww8CRJuWDgSZJywcCTJOWCgSdJygUDT5KUCwaeJCkXDDxJUi40G3gRMTYi9m6JYiRJqpRS9vB+BiyPiLsj4rSI6FDpoiRJKrdSAu9MYAbwRWAW8D8R8aOI+ExFK5MkqYyaDbyU0oyU0t8DhwO/A/YHLgEWRURNZcuTJKk8SjmGNywi7gNeA/4WmAuMBm4GvlPZ8iRJKo+OJcwzE1gN/Bz4t5TSIoCIeB74dAVrkySpbEoJvIuA21NKq4obU0ovACdWpCpJksqslEEricLAFQAi4ryIuLByJUmSVH6lBN73gE5F07sC11amHEmSKqOUwPsYhZGZmx0ARGXKkSSpMko5hjcXuDIiDqUQdF8GflvRqiRJKrNSAu9i4AHgq9n0f1E4D0+SpDaj2cBLKb2S7d0dkjW9nFLaVNmyJEkqr2YDLyKCwt7d4UDnrC2llL5d4dokSSqbUro0fwqcT+H0hM2DVRJg4EmS2oxSRmmeDvwye34x8BiFUxUkSWozSgm8fYAnKezdrQDuBc7ZmZVGRNeIuDcilkbESxHx2YjYNyL+MyJeyX7uszPrkCSpWCmB92cKXZ/LKXRv/hDYfSfX+6/AQymlvwb6Ay8B44HfpZQOpnBXhvE7uQ5JkuqVEnhXAa8ClwJrgZXsxGkJEbEXcAJwG0BKaX1K6V1gODAtm20ahfP9JEkqiyYDL7u7+d8A61NKd6eUDkwp9Ugp3bUT6/wroA74eUQsjIhbI2J34ICU0nKA7Of+Tb2JJEnbo8nAy863+zJwUBnX2RE4Avj3lNLfAO+zHd2XETE2IuZHxPy6uroyliVJas9KOS1hDvAvEdGJwnE8oHAn9B1c5zJgWUrpD9n0vRQC738jokdKaXlE9AD+0tDCKaUpwBSAgQMHph2sQZKUM6UE3j9kP2/KfgaF8/A67MgKU0p/joj/johDUkovAycBS7LHucCE7Of9O/L+kiQ1pJTAu5ZCwJXTRcD0iNgVeJ1CqH4M+FVEfA34EzCizOuUJOVYKdfSrCn3SlNKtcDABl46qdzrkiQJSruW5qMNNKeUkuEkSWozSunSHNJAm4NFJEltSimB173o+T5ADUWjNSVJagtKudJKKnq8B7xMYRSlJEltRil7eG+zbRfmyxWoRZKkiikl8J7go8DbBLwB3FipgiRJqoRSTksY0gJ1SJJUUc0ew4uI2yOipmj6moi4vaJVSZJUZqUMWjkDeLNo+k3gK5UpR5Kkyigl8N4FPlc0PYTCPfEkSWozShm0MhsYGxFfyqb3J7tbgSRJbUUpgXcZsCswNJueClxeqYIkSaqEUkZprgLOa4FaJEmqmFJGac6JiB8VTU+KiMcqW5YkSeVVyqCVQcALRdOLgKMrU44kSZVRSuD9BfhKRHSJiN2BM7M2SZLajFIGrdwJfJfChaMThZCcUMmiJEkqt1IC71+AD/holOZs4PsVq0iSpApotkszpbQhpXRtSmlQSmkQMINCCEqS1GaUsodHRPw18FVgJPDXWbOhJ0lqMxoNvIg4mELIfRU4DAgKx/D+H/CLFqlOkqQyaWoP72UKAbcc+CkwD7gduDWlNKsFapMkqWya69L8EHgceBTvci5JasOaGrTyLeAZCsftfg08R2GP76iI6NYCtUmSVDaNBl5K6Scppc8BvYFLgYXZS1cCf26B2iRJKptSTktYnlL615TSYKAPhbsnLKh4ZZIklVEplxarl1JallL6YUrpmEoVJElSJWxX4EmS1FYZeJKkXDDwJEm50OylxSLiEOA7QF+gQ9acUkonVbAuSZLKqpRrac4EDtmqLVWgFkmSKqaULs19gUlAD6B79ti/kkVJklRupQTeLcAngT0o7NltfkiS1GaU0qX5zxQCbmhRWypxWUmSWoVSQusJ3KOTJLVxzQZeSmlIC9QhSVJFlXJaQgCjgMOBzllzSil9u5KFSZJUTqV0af4UOJ9Ct2ZkbQkw8CRJbUYpozRPB36ZPb8YeAz4XsUqkiSpAkoJvH2AJyns3a0A7gXOqWRRkiSVWyldmn/O5ltOoXtzV+C9ShYlSVK5lbKHdxXwGoVjdmuBlcAllSxKkqRyK+W0hDsAIqIr0CeltK7iVUmSVGbN7uFFRN+ImAe8DRwfEY9HxLU7u+KI6BARCyPigWy6X0T8ISJeiYi7I2LXnV2HJEmbldKl+TOgF4VBKx9SuPLKqDKs+2LgpaLpHwCTUkoHA/8HfK0M65AkCSgt8AYDPymafo1CAO6wiOgFnAbcmk0H8HkKI0ABpgFf3pl1SJJUrJTAexs4LHu+P4W9u7d2cr2Tgcsp7DECdAPeTSltzKaXAT13ch2SJNUr9fZAoyh0aU4HvgDcvKMrjIihwF9SSguKmxuYtcELVkfE2IiYHxHz6+rqdrQMSVLOlDJK8/sR8RaFLsgAZqeUbt+JdR4LDIuIUylcm3MvCnt8XSOiY7aX14tG9iJTSlOAKQADBw70Lg6SpJKUsodHSmlaSumrKaUROxl2pJSuSCn1Sin1pbDn+GhK6WwKlyw7M5vtXOD+nVmPJEnFGg28iNjUxGNjY8vthO8Cl0bEqxSO6d1WgXVIknKqqS7NoHAc7S3g3UqsPKU0B5iTPX8dGFSJ9UiS1FSX5lTgfWA/4AXg0pTS4ZsfLVGcJEnl0mjgpZTOAw4Evgn0Bh6KiDci4uSWKk6SpHJpctBKSmkN8DrwR2A9hb29PVugLkmSyqqpQSv/HBGvAI8CnwQuAnqklO5pqeIkSSqXpgatXEdh0MrrFK62MozC+XMAKaU0vPLlSZJUHs2deB7AQdmjmCd8S5LalKYCr1+LVSFJUoU1GngppTdbshBJkiqppEuLSZLU1hl4kqRcMPAkSblg4EmScsHAkyTlgoEnScoFA0+SlAsGniQpFww8SVIuGHiSpFww8CRJuWDgSZJywcCTJOWCgSdJygUDT5KUCwaeJCkXDDxJUi4YeJKkXDDwJEm5YOBJknLBwJMk5YKBJ0nKBQNPkpQLBp4kKRcMPElSLhh4kqRcMPAkSblg4EmScsHAkyTlgoEnScoFA0+SlAsGniQpFww8SVIuGHiSpFww8CRJudDigRcRvSPisYh4KSJejIiLs/Z9I+I/I+KV7Oc+LV2bJKn9qsYe3kbg2ymlTwPHABdGxKHAeOB3KaWDgd9l05IklUWLB15KaXlK6bns+SrgJaAnMByYls02DfhyS9cmSWq/qnoMLyL6An8D/AE4IKW0HAqhCOxfvcokSe1N1QIvIvYAfg1cklJ6bzuWGxsR8yNifl1dXeUKlCS1K1UJvIjYhULYTU8pzcia/zciemSv9wD+0tCyKaUpKaWBKaWB3bt3b5mCJUltXjVGaQZwG/BSSulHRS/NAs7Nnp8L3N/StUmS2q+OVVjnscA5wAsRUZu1/TMwAfhVRHwN+BMwogq1SZLaqRYPvJTSU0A08vJJLVmLJCk/vNKKJCkXDDxJUi4YeJKkXDDwJEm5YOBJknLBwJMk5YKBJ0nKBQNPkpQLBp4kKRcMPElSLhh4kqRcMPAkSblg4EmScsHAkyTlgoEnScoFA0+SlAsGniQpFww8SVIuGHiSpFww8CRJuWDgSZJywcCTJOWCgSdJygUDT5KUCwaeJCkXDDxJUi4YeJKkXDDwJEm5YOBJknLBwJMk5YKBJ0nKBQNPkpQLBp4kKRcMPElSLhh4kqRcMPAkSblg4EmScsHAkyTlgoEnScoFA0+SlAsGniQpFww8SVIuGHiSpFww8CRJudCqAi8iTo6IlyPi1YgYX+16JEntR6sJvIjoAPwUOAU4FDgrIg6tblWSpPai1QQeMAh4NaX0ekppPXAXMLzKNUmS2onWFHg9gf8uml6WtUmStNMipVTtGgCIiBHAl1JKX8+mzwEGpZQu2mq+scDYbPIQ4OUWLXT77Qe8Xe0itAW3Sevkdml92uo26ZNS6r51Y8dqVNKIZUDvoulewFtbz5RSmgJMaamidlZEzE8pDax2HfqI26R1cru0Pu1tm7SmLs1ngYMjol9E7AqMAmZVuSZJUjvRavbwUkobI2Ic8DDQAfiPlNKLVS5LktROtJrAA0gpPQg8WO06yqzNdL/miNukdXK7tD7tapu0mkErkiRVUms6hidJUsUYeCWKiCsj4sWIWBQRtRFxdETMiYj5RfMMjIg5RdPHRcS8iFiaPcZm7V0j4p2IiGz6sxGRIqJXNr13RKyICLdPEyJiU7YtNj/6RsSQ7N/y74rmeyAihmTPd42IyRHxWkS8EhH3F/27T4qIS4qWezgibi2a/mFEXNqCH7HNiojVjbSPLfp9mBcRx2Xt92Xb8NWIWFm0TQc3tc1Uuu3dJtlrQyNiYUQ8HxFLIuIbpSzXaqWUfDTzAD4LzAU6ZdP7AR8H5gB/Ak7J2gcCc7LnB2avHVG0zALgtGz6ReDQ7Pm3geeAr2bTXwJ+U+3P3dofwOoG2oZQuIDB74vaHgCGZM9vBG4DOmTT/wDMAwIYAfwqa/9Ytr3mFr3PXODoan/utvBoZNsMzf5N98umj8h+Rw7cavs9sNVyjW6zan/OtvTY3m0C7ELh1LBe2WudgENK3Zat8eEeRGl6AG+nlNYBpJTeTiltPkdwInBVA8tcCExNKT23eRngcmDzRbGfBgZnzwcDk7aafqbcHyJHngdWRsQXihsjoguFP5b/lFLaBJBS+jmwDvg8W26TzwCLgVURsU9EdAI+DSxsmY/QLn0XuCz7XSD73ZhG4XelQSVsM+2cprbJnhQGNr6TvbYupfRyCcu1WgZeaR4BekfEf0XEv0XE54pemwusi4gTt1rmMxS+ARWbn7VDIdA2/3H9K+AeCnuIZO1Pl6v4dmy3oq6v+7Z67Tq2/SLySeBPKaX3tmqfD3wm+xKzMSI+QWEbzAX+QGEPfyCwKBWu86od09zvREOa3GZlrC2vGt0mKaUVFM6FfjMi7oyIs4sOs+zItqw6A68EKaXVwJEULmlWB9wdEWOKZmnoj2sADQ2B3dz2NDA4IvoBb6SU1gIREXtk65pXvk/Qbn2QUhqQPU4vfiGl9CRARBxf1NzYNilu37yXtznw5hZNu9ddfo1tk+Zeb2457bj6f9tUuNTjSRT+Hn0H+I9SlmutDLwSpZQ2pZTmpJSuBsYBZxS99ijQGTimaJEX+WiPbbMjgSXZMq8A+wB/R+GPKhS+Mf0D8McsZLVzrgeuLJp+FegTEXtuNd8RZNuFj/a8D6fQpfl7Cnt47nXvvCUUfgeKFf/bN6SUbaYd1+w2SSm9kFKaBHyBj/7u7ci2rDoDrwQRcUhEHFzUNAB4c6vZrqdwjG6znwJjImJA9h7dgB8ANxTNMxe4mI8Cby5wCe5JlEVK6REKXyr6Z9PvUzjO8KPs/otExGigC/BottjTFA7Ir8i+5KwAuvLRwCXtuBuAH2S/C2S/G2OAf2tsgRK3mXZco9skIvbYPLo5U/x3b7u3ZWvQqq600ortAfw4IroCGyl86xwL3Lt5hpTSgxFRVzS9PCL+Hrgl+3YawOSU0uyi930aOJVC3zcU/qD+FQZeOV0P3F80fQWFUX//FREfAkuB01M21Ax4gcKI2l8WLfMCsMfmA/QqSZeIWFY0/aOU0o8ioifwTEQkYBXw9yml5c28V3PbTKXZrm2S/d26PCJuBj4A3qcQaqSUZu3gtqwqr7QiScoFuzQlSblg4EmScsHAkyTlgoEnScoFA0+SlAsGniQpFww8SVIuGHiSpFz4/ymEnrEJzkixAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np \n",
    "import matplotlib.pyplot as plt \n",
    "   \n",
    "# set width of bar \n",
    "barWidth = 0.25\n",
    "fig = plt.subplots(figsize =(7, 5)) \n",
    "   \n",
    "# set height of bar \n",
    "boys = [90.03, 87.78, 83.41, 84.52]\n",
    "girls = [98.56, 97.65, 85.92, 86.62] \n",
    "#CSE = [29, 3, 24, 25, 17] \n",
    "   \n",
    "# Set position of bar on X axis \n",
    "br1 = np.arange(4) \n",
    "br2 = [x + barWidth for x in br1] \n",
    "#br3 = [x + barWidth for x in br2] \n",
    "   \n",
    "# Make the plot \n",
    "p1=plt.bar(br1, boys, color ='r', width = barWidth, \n",
    "        edgecolor ='grey', label ='Jordao, Artur, et al. [1]') \n",
    "p2=plt.bar(br2, girls, color ='b', width = barWidth, \n",
    "        edgecolor ='grey', label ='Bonsai') \n",
    "#plt.bar(br3, CSE, color ='b', width = barWidth, \n",
    "#        edgecolor ='grey', label ='CSE') \n",
    "   \n",
    "# Adding Xticks  \n",
    "plt.xlabel('', fontweight ='bold') \n",
    "plt.ylabel('Mean Accuracy', fontweight ='bold') \n",
    "plt.xticks([r + 0.1 for r in range(len(boys))], \n",
    "           ['SNOW', 'FNOW', 'LOTO', 'LOSO']) \n",
    "plt.legend((p1[0], p2[0]), ('Jordao, Artur, et al. [1]', 'Bonsai')) \n",
    "#plt.title('Comparison of Mean Accuracies with benchmark paper ') \n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy[1.0000] Recall[1.0000] F1[1.0000] at fold[0]\n",
      "______________________________________________________\n",
      "Accuracy[0.8403] Recall[0.8579] F1[0.8405] at fold[1]\n",
      "______________________________________________________\n",
      "Accuracy[0.9442] Recall[0.9476] F1[0.9444] at fold[2]\n",
      "______________________________________________________\n",
      "Accuracy[0.7820] Recall[0.7721] F1[0.7517] at fold[3]\n",
      "______________________________________________________\n",
      "Accuracy[0.8340] Recall[0.8406] F1[0.7990] at fold[4]\n",
      "______________________________________________________\n",
      "Accuracy[0.9129] Recall[0.9167] F1[0.8930] at fold[5]\n",
      "______________________________________________________\n",
      "Accuracy[0.8182] Recall[0.8261] F1[0.7740] at fold[6]\n",
      "______________________________________________________\n",
      "Accuracy[0.8452] Recall[0.8659] F1[0.8334] at fold[7]\n",
      "______________________________________________________\n",
      "Accuracy[0.9721] Recall[0.9716] F1[0.9711] at fold[8]\n",
      "______________________________________________________\n",
      "Accuracy[0.9395] Recall[0.9453] F1[0.9360] at fold[9]\n",
      "______________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#from bonsai.ensemble.streamforests import StreamForests\n",
    "avg_acc = []\n",
    "avg_recall = []\n",
    "avg_f1 = []\n",
    "avg_ttime=[]\n",
    "avg_ptime=[]\n",
    "avg_size=[]\n",
    "for i in range(0, len(folds)):\n",
    "    train_idx = folds[i][0]\n",
    "    test_idx = folds[i][1]\n",
    "\n",
    "    X_train, y_train = X[train_idx], y[train_idx]\n",
    "    X_test, y_test = X[test_idx], y[test_idx]\n",
    "\n",
    "        #Your train goes here. For instance:\n",
    "    #X_train=X_train.transpose(0,1,2).reshape(X_train.shape[0],-1)\n",
    "    #X_test=X_test.transpose(0,1,2).reshape(X_test.shape[0],-1)\n",
    "    X_train = feature_extraction(X_train)\n",
    "    X_test = feature_extraction(X_test)      \n",
    "    reg_params = {\"max_depth\":10,\"min_samples_split\":1,\"min_samples_leaf\":1}\n",
    "    method = StreamForests(base_estimator=XGBTree,base_params=reg_params,n_estimators=10)\n",
    "    t0=time.time()\n",
    "    method.fit(X_train, y_train)\n",
    "    avg_ttime.append(time.time()-t0)\n",
    "            #Your testing goes here. For instance:\n",
    "    t1=time.time()\n",
    "    y_pred = method.predict(X_test)\n",
    "    avg_ptime.append(time.time()-t1)\n",
    "    y_pred=np.round(y_pred,0)\n",
    "    y_pred=y_pred.astype(int)\n",
    "    v=method.dump()\n",
    "    avg_size.append(round(v.__sizeof__()/1024,3))\n",
    "\n",
    "    acc_fold = accuracy_score(y_test, y_pred)\n",
    "    avg_acc.append(acc_fold)\n",
    "\n",
    "    recall_fold = recall_score(y_test, y_pred, average='macro')\n",
    "    avg_recall.append(recall_fold)\n",
    "\n",
    "    f1_fold  = f1_score(y_test, y_pred, average='macro')\n",
    "    avg_f1.append(f1_fold)\n",
    "\n",
    "    print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.format(acc_fold, recall_fold, f1_fold ,i))\n",
    "    print('______________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Accuracy[0.8888] IC [0.8460, 0.9317]\n",
      "Mean Recall[0.8944] IC [0.8522, 0.9366]\n",
      "Mean F1[0.8743] IC [0.8240, 0.9246]\n",
      "Mean size[0.164]\n",
      "Mean training time[10734.799]\n",
      "Mean prediction time[1.295]\n"
     ]
    }
   ],
   "source": [
    "ic_acc = st.t.interval(0.9, len(avg_acc) - 1, loc=np.mean(avg_acc), scale=st.sem(avg_acc))\n",
    "ic_recall = st.t.interval(0.9, len(avg_recall) - 1, loc=np.mean(avg_recall), scale=st.sem(avg_recall))\n",
    "ic_f1 = st.t.interval(0.9, len(avg_f1) - 1, loc=np.mean(avg_f1), scale=st.sem(avg_f1))\n",
    "print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_acc), ic_acc[0], ic_acc[1]))\n",
    "print('Mean Recall[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_recall), ic_recall[0], ic_recall[1]))\n",
    "print('Mean F1[{:.4f}] IC [{:.4f}, {:.4f}]'.format(np.mean(avg_f1), ic_f1[0], ic_f1[1]))\n",
    "print('Mean size[{:.3f}]'.format(np.mean(avg_size)))\n",
    "print('Mean training time[{:.3f}]'.format(round(np.mean(avg_ttime)*1000,3)))\n",
    "print('Mean prediction time[{:.3f}]'.format(round(np.mean(avg_ptime)*1000,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StreamForests():\n",
    "\n",
    "    def __init__(self,\n",
    "                base_estimator,\n",
    "                base_params,\n",
    "                n_estimators=100):\n",
    "        self.base_estimator = base_estimator\n",
    "        self.base_params = base_params.copy()\n",
    "        self.n_estimators = n_estimators\n",
    "        self.estimators = []\n",
    "\n",
    "    def fit(self, X, y):\n",
    "\n",
    "        X = X.astype(np.float)\n",
    "        y = y.astype(np.float)\n",
    "        if \"random_state\" not in self.base_params:\n",
    "            self.base_params[\"random_state\"] = 1\n",
    " \n",
    "        bonsai_tmp = RegTree()\n",
    "        bonsai_tmp.init_cnvs(X)\n",
    "        xdim, cnvs, cnvsn = bonsai_tmp.get_cnvs()\n",
    "        #print(\"hell\")\n",
    "        for i in range(self.n_estimators):\n",
    "            self.base_params[\"random_state\"] += 1\n",
    "            estimator = self.base_estimator(**self.base_params)\n",
    "            estimator.set_cnvs(xdim, cnvs, cnvsn)\n",
    "            estimator.fit(X, y, init_cnvs=False)\n",
    "            self.estimators.append(estimator)\n",
    "\n",
    "    def predict(self, X):\n",
    "        n, m = X.shape\n",
    "        y_hat = np.zeros(n) \n",
    "        k = len(self.estimators)\n",
    "        for estimator in self.estimators:\n",
    "            y_hat += estimator.predict(X) \n",
    "        y_hat /= k\n",
    "        return y_hat\n",
    "\n",
    "    def dump(self, columns=[]): \n",
    "        return [estimator.dump(columns) \n",
    "                for estimator in self.estimators]\n",
    "      \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}