Catboost/CatBoostRegressionTutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python Catboost Tutorial - Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adapted from the Catboost repository."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CatBoost installation\n",
    "If you have not already installed CatBoost: <br>\n",
    "pip install --upgrade catboost\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostRegressor, Pool, cv\n",
    "from catboost.eval.catboost_evaluation import *\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "from itertools import product\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "from imblearn.over_sampling import SMOTE, SMOTENC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define function to calculate the MAPE\n",
    "def mean_absolute_percentage_error(y_true, y_pred): \n",
    "    y_true, y_pred = np.array(y_true), np.array(y_pred)\n",
    "    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DF shape (891, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass     Sex   Age  SibSp  Parch     Fare Cabin Embarked\n",
       "0         0       3    male  22.0      1      0   7.2500   NaN        S\n",
       "1         1       1  female  38.0      1      0  71.2833   C85        C\n",
       "2         1       3  female  26.0      0      0   7.9250   NaN        S\n",
       "3         1       1  female  35.0      1      0  53.1000  C123        S\n",
       "4         0       3    male  35.0      0      0   8.0500   NaN        S"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Import Data\n",
    "df = pd.read_csv(\"https://raw.githubusercontent.com/iandreafc/sna-bigdata-course/master/Datasets/titanic.csv\")\n",
    "\n",
    "#See the imported dataset\n",
    "print(\"DF shape\", df.shape)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Preparation\n",
    "First of all let's check how many missing values do we have:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Survived      0\n",
       "Pclass        0\n",
       "Sex           0\n",
       "Age         177\n",
       "SibSp         0\n",
       "Parch         0\n",
       "Fare          0\n",
       "Cabin       687\n",
       "Embarked      2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we cat see, **`Age`**, **`Cabin`** and **`Embarked`** indeed have some missing values, so let's fill them with some number way out of their distributions - so the model would be able to easily distinguish between them and take it into account:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Survived    0\n",
       "Pclass      0\n",
       "Sex         0\n",
       "Age         0\n",
       "SibSp       0\n",
       "Parch       0\n",
       "Fare        0\n",
       "Cabin       0\n",
       "Embarked    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(-999, inplace=True)\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's separate features and label variable, **to test regression we try to predict the ticket price**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop('Fare', axis=1)\n",
    "y = df.Fare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pay attention that our features are of differnt types - some of them are numeric, some are categorical, and some are even just strings, which normally should be handled in some specific way (for example encoded with bag-of-words representation). But in our case we could treat these string features just as categorical one - all the heavy lifting is done inside CatBoost. How cool is that? :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Survived      int64\n",
      "Pclass        int64\n",
      "Sex          object\n",
      "Age         float64\n",
      "SibSp         int64\n",
      "Parch         int64\n",
      "Cabin        object\n",
      "Embarked     object\n",
      "dtype: object\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 4, 5, 6, 7], dtype=int64)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X.dtypes)\n",
    "\n",
    "categorical_features_indices = np.where(X.dtypes != np.float)[0]\n",
    "categorical_features_indices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Encode Strings\n",
    "Not strictly necessary in Catboost, but useful for example for SMOTE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>82</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>56</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass  Sex   Age  SibSp  Parch  Cabin  Embarked\n",
       "0         0       3    1  22.0      1      0      0         3\n",
       "1         1       1    0  38.0      1      0     82         1\n",
       "2         1       3    0  26.0      0      0      0         3\n",
       "3         1       1    0  35.0      1      0     56         3\n",
       "4         0       3    1  35.0      0      0      0         3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for var in ['Sex', 'Cabin', 'Embarked']:\n",
    "    X[var] = X[var].astype('category').cat.codes\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Splitting\n",
    "Let's split the train data into training and validation sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameters Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Combinations: 60\n"
     ]
    }
   ],
   "source": [
    "#Define a grid of parameters to test\n",
    "grid = {'learning_rate': [0.01, 0.03, 0.1, 0.2],\n",
    "        'depth': [4, 6, 10],\n",
    "        'l2_leaf_reg': [1, 3, 5, 7, 9],\n",
    "        }\n",
    "\n",
    "#Count all possible combinations\n",
    "print(\"# Combinations:\", len([dict(zip(grid.keys(),v)) for v in product(*grid.values())]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tloss: 51.4257794\tbest: 51.4257794 (0)\ttotal: 3.71s\tremaining: 3m 38s\n",
      "1:\tloss: 48.5511426\tbest: 48.5511426 (1)\ttotal: 7.91s\tremaining: 3m 49s\n",
      "2:\tloss: 47.6622629\tbest: 47.6622629 (2)\ttotal: 14.6s\tremaining: 4m 36s\n",
      "3:\tloss: 46.5503441\tbest: 46.5503441 (3)\ttotal: 21.6s\tremaining: 5m 2s\n",
      "4:\tloss: 53.2308382\tbest: 46.5503441 (3)\ttotal: 24.4s\tremaining: 4m 28s\n",
      "5:\tloss: 44.8138739\tbest: 44.8138739 (5)\ttotal: 27.9s\tremaining: 4m 10s\n",
      "6:\tloss: 47.2186616\tbest: 44.8138739 (5)\ttotal: 33.1s\tremaining: 4m 10s\n",
      "7:\tloss: 42.1276775\tbest: 42.1276775 (7)\ttotal: 39.5s\tremaining: 4m 17s\n",
      "8:\tloss: 55.2727481\tbest: 42.1276775 (7)\ttotal: 42.1s\tremaining: 3m 58s\n",
      "9:\tloss: 47.4190118\tbest: 42.1276775 (7)\ttotal: 44.9s\tremaining: 3m 44s\n",
      "10:\tloss: 45.5511488\tbest: 42.1276775 (7)\ttotal: 49.3s\tremaining: 3m 39s\n",
      "11:\tloss: 45.0541410\tbest: 42.1276775 (7)\ttotal: 55.6s\tremaining: 3m 42s\n",
      "12:\tloss: 56.4005909\tbest: 42.1276775 (7)\ttotal: 58s\tremaining: 3m 29s\n",
      "13:\tloss: 51.9701344\tbest: 42.1276775 (7)\ttotal: 1m\tremaining: 3m 18s\n",
      "14:\tloss: 42.2515032\tbest: 42.1276775 (7)\ttotal: 1m 4s\tremaining: 3m 14s\n",
      "15:\tloss: 42.9094850\tbest: 42.1276775 (7)\ttotal: 1m 10s\tremaining: 3m 12s\n",
      "16:\tloss: 56.2533464\tbest: 42.1276775 (7)\ttotal: 1m 12s\tremaining: 3m 3s\n",
      "17:\tloss: 50.7947224\tbest: 42.1276775 (7)\ttotal: 1m 14s\tremaining: 2m 54s\n",
      "18:\tloss: 46.9143331\tbest: 42.1276775 (7)\ttotal: 1m 18s\tremaining: 2m 50s\n",
      "19:\tloss: 43.4335473\tbest: 42.1276775 (7)\ttotal: 1m 24s\tremaining: 2m 49s\n",
      "20:\tloss: 52.3425617\tbest: 42.1276775 (7)\ttotal: 1m 32s\tremaining: 2m 52s\n",
      "21:\tloss: 49.3638899\tbest: 42.1276775 (7)\ttotal: 1m 44s\tremaining: 2m 59s\n",
      "22:\tloss: 46.6094439\tbest: 42.1276775 (7)\ttotal: 1m 58s\tremaining: 3m 10s\n",
      "23:\tloss: 50.8888776\tbest: 42.1276775 (7)\ttotal: 2m 13s\tremaining: 3m 20s\n",
      "24:\tloss: 53.2766404\tbest: 42.1276775 (7)\ttotal: 2m 21s\tremaining: 3m 18s\n",
      "25:\tloss: 49.0860061\tbest: 42.1276775 (7)\ttotal: 2m 30s\tremaining: 3m 17s\n",
      "26:\tloss: 43.6843498\tbest: 42.1276775 (7)\ttotal: 2m 44s\tremaining: 3m 20s\n",
      "27:\tloss: 47.2128872\tbest: 42.1276775 (7)\ttotal: 2m 58s\tremaining: 3m 24s\n",
      "28:\tloss: 55.1993529\tbest: 42.1276775 (7)\ttotal: 3m 5s\tremaining: 3m 18s\n",
      "29:\tloss: 45.4735991\tbest: 42.1276775 (7)\ttotal: 3m 13s\tremaining: 3m 13s\n",
      "30:\tloss: 43.0808194\tbest: 42.1276775 (7)\ttotal: 3m 26s\tremaining: 3m 12s\n",
      "31:\tloss: 45.7753291\tbest: 42.1276775 (7)\ttotal: 3m 40s\tremaining: 3m 13s\n",
      "32:\tloss: 56.4231060\tbest: 42.1276775 (7)\ttotal: 3m 46s\tremaining: 3m 5s\n",
      "33:\tloss: 46.7407281\tbest: 42.1276775 (7)\ttotal: 3m 52s\tremaining: 2m 58s\n",
      "34:\tloss: 42.0325558\tbest: 42.0325558 (34)\ttotal: 4m 4s\tremaining: 2m 54s\n",
      "35:\tloss: 42.4197039\tbest: 42.0325558 (34)\ttotal: 4m 17s\tremaining: 2m 51s\n",
      "36:\tloss: 56.7761304\tbest: 42.0325558 (34)\ttotal: 4m 23s\tremaining: 2m 43s\n",
      "37:\tloss: 47.1515697\tbest: 42.0325558 (34)\ttotal: 4m 30s\tremaining: 2m 36s\n",
      "38:\tloss: 41.4788404\tbest: 41.4788404 (38)\ttotal: 4m 41s\tremaining: 2m 31s\n",
      "39:\tloss: 46.4909448\tbest: 41.4788404 (38)\ttotal: 4m 55s\tremaining: 2m 27s\n",
      "40:\tloss: 54.9358058\tbest: 41.4788404 (38)\ttotal: 5m 11s\tremaining: 2m 24s\n",
      "41:\tloss: 53.7037959\tbest: 41.4788404 (38)\ttotal: 5m 34s\tremaining: 2m 23s\n",
      "42:\tloss: 53.3109405\tbest: 41.4788404 (38)\ttotal: 6m 1s\tremaining: 2m 23s\n",
      "43:\tloss: 51.5945058\tbest: 41.4788404 (38)\ttotal: 6m 30s\tremaining: 2m 21s\n",
      "44:\tloss: 56.3255701\tbest: 41.4788404 (38)\ttotal: 6m 45s\tremaining: 2m 15s\n",
      "45:\tloss: 54.0434372\tbest: 41.4788404 (38)\ttotal: 7m 7s\tremaining: 2m 10s\n",
      "46:\tloss: 52.7798111\tbest: 41.4788404 (38)\ttotal: 7m 34s\tremaining: 2m 5s\n",
      "47:\tloss: 54.5052848\tbest: 41.4788404 (38)\ttotal: 8m 3s\tremaining: 2m\n",
      "48:\tloss: 57.2658689\tbest: 41.4788404 (38)\ttotal: 8m 17s\tremaining: 1m 51s\n",
      "49:\tloss: 54.8089332\tbest: 41.4788404 (38)\ttotal: 8m 37s\tremaining: 1m 43s\n",
      "50:\tloss: 51.1661094\tbest: 41.4788404 (38)\ttotal: 9m 5s\tremaining: 1m 36s\n",
      "51:\tloss: 46.8753777\tbest: 41.4788404 (38)\ttotal: 9m 34s\tremaining: 1m 28s\n",
      "52:\tloss: 57.4639822\tbest: 41.4788404 (38)\ttotal: 9m 48s\tremaining: 1m 17s\n",
      "53:\tloss: 51.6891407\tbest: 41.4788404 (38)\ttotal: 10m 6s\tremaining: 1m 7s\n",
      "54:\tloss: 49.9596598\tbest: 41.4788404 (38)\ttotal: 10m 31s\tremaining: 57.4s\n",
      "55:\tloss: 50.0066571\tbest: 41.4788404 (38)\ttotal: 10m 58s\tremaining: 47s\n",
      "56:\tloss: 57.4656330\tbest: 41.4788404 (38)\ttotal: 11m 10s\tremaining: 35.3s\n",
      "57:\tloss: 53.3299965\tbest: 41.4788404 (38)\ttotal: 11m 27s\tremaining: 23.7s\n",
      "58:\tloss: 50.8113985\tbest: 41.4788404 (38)\ttotal: 11m 51s\tremaining: 12.1s\n",
      "59:\tloss: 49.6141098\tbest: 41.4788404 (38)\ttotal: 12m 17s\tremaining: 0us\n",
      "Estimating final quality...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'depth': 6, 'l2_leaf_reg': 9, 'learning_rate': 0.1}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Define Model\n",
    "model = CatBoostRegressor()\n",
    "\n",
    "#Use the Pool function to specify the categorical features\n",
    "train_pool = Pool(data= X_train, label= y_train, cat_features= categorical_features_indices)\n",
    "\n",
    "#Grid Search\n",
    "#Default cross-validation is 3-fold\n",
    "grid_search_result = model.grid_search(grid, train_pool, cv=3)\n",
    "bestparam = grid_search_result[\"params\"]\n",
    "bestparam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss_function': 'RMSE', 'depth': 6, 'l2_leaf_reg': 9, 'learning_rate': 0.1, 'eval_metric': 'RMSE'}\n"
     ]
    }
   ],
   "source": [
    "#Set best params\n",
    "model = CatBoostRegressor()\n",
    "\n",
    "#Depending on your objective you can also customize the evaluation metric\n",
    "bestparam[\"eval_metric\"] = \"RMSE\"\n",
    "\n",
    "model.set_params(**bestparam)\n",
    "print(model.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Training\n",
    "Retaining the best model and with early stopping, to avoid overfit.\n",
    "**In real cases, we need an external test set, not used for training or validation (early stopping). That dataset is the one to be used to evaluate the final moldel.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(668, 8) (501, 8) (167, 8)\n"
     ]
    }
   ],
   "source": [
    "#Furter split the train set into final_train and validation sets\n",
    "X_train_final, X_validation, y_train_final, y_validation = train_test_split(X_train, y_train,\\\n",
    "                                                                            train_size=0.75, random_state=14)\n",
    "\n",
    "print(X_train.shape, X_train_final.shape, X_validation.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use early sotopping rounds and validation set, to stop after K iterations with no improvement of the evaluation metric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 53.4933716\ttest: 36.0313344\tbest: 36.0313344 (0)\ttotal: 43.2ms\tremaining: 43.2s\n",
      "1:\tlearn: 51.9761312\ttest: 34.4858794\tbest: 34.4858794 (1)\ttotal: 50.1ms\tremaining: 25s\n",
      "2:\tlearn: 50.6086357\ttest: 33.1763546\tbest: 33.1763546 (2)\ttotal: 64.7ms\tremaining: 21.5s\n",
      "3:\tlearn: 49.5492064\ttest: 32.1721324\tbest: 32.1721324 (3)\ttotal: 73.1ms\tremaining: 18.2s\n",
      "4:\tlearn: 48.4987617\ttest: 31.0057940\tbest: 31.0057940 (4)\ttotal: 84.1ms\tremaining: 16.7s\n",
      "5:\tlearn: 47.4528906\ttest: 30.1281782\tbest: 30.1281782 (5)\ttotal: 90.5ms\tremaining: 15s\n",
      "6:\tlearn: 46.6617979\ttest: 29.7176684\tbest: 29.7176684 (6)\ttotal: 99.2ms\tremaining: 14.1s\n",
      "7:\tlearn: 45.9528926\ttest: 29.3022339\tbest: 29.3022339 (7)\ttotal: 113ms\tremaining: 14s\n",
      "8:\tlearn: 45.2723747\ttest: 28.6544811\tbest: 28.6544811 (8)\ttotal: 120ms\tremaining: 13.2s\n",
      "9:\tlearn: 44.6471027\ttest: 28.1572765\tbest: 28.1572765 (9)\ttotal: 126ms\tremaining: 12.5s\n",
      "10:\tlearn: 44.1531113\ttest: 27.6852475\tbest: 27.6852475 (10)\ttotal: 129ms\tremaining: 11.6s\n",
      "11:\tlearn: 43.6548193\ttest: 27.2431684\tbest: 27.2431684 (11)\ttotal: 140ms\tremaining: 11.5s\n",
      "12:\tlearn: 43.2637856\ttest: 26.9389250\tbest: 26.9389250 (12)\ttotal: 150ms\tremaining: 11.4s\n",
      "13:\tlearn: 42.9061195\ttest: 26.6065301\tbest: 26.6065301 (13)\ttotal: 155ms\tremaining: 10.9s\n",
      "14:\tlearn: 42.5446691\ttest: 26.3455209\tbest: 26.3455209 (14)\ttotal: 161ms\tremaining: 10.6s\n",
      "15:\tlearn: 42.2092496\ttest: 26.1518452\tbest: 26.1518452 (15)\ttotal: 165ms\tremaining: 10.1s\n",
      "16:\tlearn: 41.9139323\ttest: 25.9582847\tbest: 25.9582847 (16)\ttotal: 169ms\tremaining: 9.75s\n",
      "17:\tlearn: 41.5577663\ttest: 25.7646177\tbest: 25.7646177 (17)\ttotal: 178ms\tremaining: 9.74s\n",
      "18:\tlearn: 41.1711730\ttest: 25.5385857\tbest: 25.5385857 (18)\ttotal: 182ms\tremaining: 9.4s\n",
      "19:\tlearn: 40.8211096\ttest: 25.5545404\tbest: 25.5385857 (18)\ttotal: 188ms\tremaining: 9.2s\n",
      "20:\tlearn: 40.6107076\ttest: 25.4986540\tbest: 25.4986540 (20)\ttotal: 194ms\tremaining: 9.03s\n",
      "21:\tlearn: 40.4570388\ttest: 25.4364496\tbest: 25.4364496 (21)\ttotal: 200ms\tremaining: 8.9s\n",
      "22:\tlearn: 40.3961219\ttest: 25.4889697\tbest: 25.4364496 (21)\ttotal: 202ms\tremaining: 8.57s\n",
      "23:\tlearn: 40.1975900\ttest: 25.3554816\tbest: 25.3554816 (23)\ttotal: 232ms\tremaining: 9.43s\n",
      "24:\tlearn: 39.9925281\ttest: 25.2977153\tbest: 25.2977153 (24)\ttotal: 239ms\tremaining: 9.3s\n",
      "25:\tlearn: 39.8319409\ttest: 25.2604084\tbest: 25.2604084 (25)\ttotal: 241ms\tremaining: 9.04s\n",
      "26:\tlearn: 39.7035207\ttest: 25.2216133\tbest: 25.2216133 (26)\ttotal: 248ms\tremaining: 8.94s\n",
      "27:\tlearn: 39.5552703\ttest: 25.1277691\tbest: 25.1277691 (27)\ttotal: 256ms\tremaining: 8.88s\n",
      "28:\tlearn: 39.4388543\ttest: 25.2158182\tbest: 25.1277691 (27)\ttotal: 265ms\tremaining: 8.87s\n",
      "29:\tlearn: 39.0969476\ttest: 25.2926902\tbest: 25.1277691 (27)\ttotal: 271ms\tremaining: 8.76s\n",
      "30:\tlearn: 39.0018461\ttest: 25.2511853\tbest: 25.1277691 (27)\ttotal: 274ms\tremaining: 8.55s\n",
      "31:\tlearn: 38.5108385\ttest: 25.3721558\tbest: 25.1277691 (27)\ttotal: 288ms\tremaining: 8.71s\n",
      "32:\tlearn: 38.2937476\ttest: 25.1823794\tbest: 25.1277691 (27)\ttotal: 291ms\tremaining: 8.53s\n",
      "33:\tlearn: 38.1682300\ttest: 25.1947248\tbest: 25.1277691 (27)\ttotal: 296ms\tremaining: 8.42s\n",
      "34:\tlearn: 38.0816567\ttest: 25.1873147\tbest: 25.1277691 (27)\ttotal: 299ms\tremaining: 8.25s\n",
      "35:\tlearn: 38.0031484\ttest: 25.1874451\tbest: 25.1277691 (27)\ttotal: 301ms\tremaining: 8.07s\n",
      "36:\tlearn: 37.9181552\ttest: 25.1682727\tbest: 25.1277691 (27)\ttotal: 305ms\tremaining: 7.94s\n",
      "37:\tlearn: 37.8235678\ttest: 25.1721385\tbest: 25.1277691 (27)\ttotal: 309ms\tremaining: 7.82s\n",
      "38:\tlearn: 37.6084292\ttest: 25.3783484\tbest: 25.1277691 (27)\ttotal: 316ms\tremaining: 7.79s\n",
      "39:\tlearn: 37.5184689\ttest: 25.4290975\tbest: 25.1277691 (27)\ttotal: 322ms\tremaining: 7.73s\n",
      "40:\tlearn: 37.2689706\ttest: 25.7119720\tbest: 25.1277691 (27)\ttotal: 326ms\tremaining: 7.63s\n",
      "41:\tlearn: 37.2363493\ttest: 25.7135360\tbest: 25.1277691 (27)\ttotal: 330ms\tremaining: 7.52s\n",
      "42:\tlearn: 37.1028317\ttest: 25.7398656\tbest: 25.1277691 (27)\ttotal: 333ms\tremaining: 7.41s\n",
      "43:\tlearn: 37.0932695\ttest: 25.7518191\tbest: 25.1277691 (27)\ttotal: 335ms\tremaining: 7.28s\n",
      "44:\tlearn: 36.9870373\ttest: 25.7023480\tbest: 25.1277691 (27)\ttotal: 341ms\tremaining: 7.25s\n",
      "45:\tlearn: 36.9671493\ttest: 25.7121208\tbest: 25.1277691 (27)\ttotal: 345ms\tremaining: 7.15s\n",
      "46:\tlearn: 36.9528393\ttest: 25.7140070\tbest: 25.1277691 (27)\ttotal: 346ms\tremaining: 7.01s\n",
      "47:\tlearn: 36.9516344\ttest: 25.7188822\tbest: 25.1277691 (27)\ttotal: 347ms\tremaining: 6.87s\n",
      "48:\tlearn: 36.8531692\ttest: 25.8654216\tbest: 25.1277691 (27)\ttotal: 358ms\tremaining: 6.96s\n",
      "49:\tlearn: 36.2403191\ttest: 26.0197459\tbest: 25.1277691 (27)\ttotal: 373ms\tremaining: 7.09s\n",
      "50:\tlearn: 36.1831981\ttest: 26.0267491\tbest: 25.1277691 (27)\ttotal: 375ms\tremaining: 6.98s\n",
      "51:\tlearn: 36.1416299\ttest: 26.0452675\tbest: 25.1277691 (27)\ttotal: 377ms\tremaining: 6.88s\n",
      "52:\tlearn: 36.0437409\ttest: 26.0885869\tbest: 25.1277691 (27)\ttotal: 383ms\tremaining: 6.84s\n",
      "53:\tlearn: 36.0307717\ttest: 26.0892003\tbest: 25.1277691 (27)\ttotal: 384ms\tremaining: 6.74s\n",
      "54:\tlearn: 36.0299850\ttest: 26.0929518\tbest: 25.1277691 (27)\ttotal: 386ms\tremaining: 6.62s\n",
      "55:\tlearn: 36.0154087\ttest: 26.0969834\tbest: 25.1277691 (27)\ttotal: 387ms\tremaining: 6.53s\n",
      "56:\tlearn: 35.9764618\ttest: 26.0812216\tbest: 25.1277691 (27)\ttotal: 389ms\tremaining: 6.44s\n",
      "57:\tlearn: 35.8279025\ttest: 26.2142508\tbest: 25.1277691 (27)\ttotal: 395ms\tremaining: 6.42s\n",
      "58:\tlearn: 35.3289550\ttest: 26.3681203\tbest: 25.1277691 (27)\ttotal: 402ms\tremaining: 6.41s\n",
      "59:\tlearn: 35.2580020\ttest: 26.3766580\tbest: 25.1277691 (27)\ttotal: 451ms\tremaining: 7.06s\n",
      "60:\tlearn: 35.1994160\ttest: 26.3131479\tbest: 25.1277691 (27)\ttotal: 476ms\tremaining: 7.32s\n",
      "61:\tlearn: 35.1171933\ttest: 26.3412799\tbest: 25.1277691 (27)\ttotal: 495ms\tremaining: 7.49s\n",
      "62:\tlearn: 35.1167223\ttest: 26.3442368\tbest: 25.1277691 (27)\ttotal: 500ms\tremaining: 7.43s\n",
      "63:\tlearn: 35.1140367\ttest: 26.3543698\tbest: 25.1277691 (27)\ttotal: 504ms\tremaining: 7.37s\n",
      "64:\tlearn: 35.1124469\ttest: 26.3624120\tbest: 25.1277691 (27)\ttotal: 518ms\tremaining: 7.45s\n",
      "65:\tlearn: 34.7484371\ttest: 26.5096990\tbest: 25.1277691 (27)\ttotal: 541ms\tremaining: 7.66s\n",
      "66:\tlearn: 34.7125723\ttest: 26.5405431\tbest: 25.1277691 (27)\ttotal: 550ms\tremaining: 7.65s\n",
      "67:\tlearn: 34.4964746\ttest: 26.5052666\tbest: 25.1277691 (27)\ttotal: 571ms\tremaining: 7.83s\n",
      "68:\tlearn: 34.4942080\ttest: 26.5148226\tbest: 25.1277691 (27)\ttotal: 572ms\tremaining: 7.72s\n",
      "69:\tlearn: 34.4686120\ttest: 26.4861451\tbest: 25.1277691 (27)\ttotal: 575ms\tremaining: 7.63s\n",
      "70:\tlearn: 34.3835017\ttest: 26.5110885\tbest: 25.1277691 (27)\ttotal: 605ms\tremaining: 7.91s\n",
      "71:\tlearn: 34.3740540\ttest: 26.5108150\tbest: 25.1277691 (27)\ttotal: 606ms\tremaining: 7.81s\n",
      "72:\tlearn: 34.3207480\ttest: 26.5098511\tbest: 25.1277691 (27)\ttotal: 611ms\tremaining: 7.76s\n",
      "73:\tlearn: 34.3204562\ttest: 26.5121731\tbest: 25.1277691 (27)\ttotal: 612ms\tremaining: 7.66s\n",
      "74:\tlearn: 33.9263415\ttest: 26.6783391\tbest: 25.1277691 (27)\ttotal: 636ms\tremaining: 7.84s\n",
      "75:\tlearn: 33.8903105\ttest: 26.6735815\tbest: 25.1277691 (27)\ttotal: 667ms\tremaining: 8.11s\n",
      "76:\tlearn: 33.8884216\ttest: 26.6777052\tbest: 25.1277691 (27)\ttotal: 669ms\tremaining: 8.02s\n",
      "77:\tlearn: 33.8869891\ttest: 26.6853099\tbest: 25.1277691 (27)\ttotal: 670ms\tremaining: 7.92s\n",
      "78:\tlearn: 33.8617830\ttest: 26.6776897\tbest: 25.1277691 (27)\ttotal: 672ms\tremaining: 7.84s\n",
      "79:\tlearn: 33.5835395\ttest: 26.7496298\tbest: 25.1277691 (27)\ttotal: 696ms\tremaining: 8.01s\n",
      "80:\tlearn: 33.5243080\ttest: 26.7165408\tbest: 25.1277691 (27)\ttotal: 702ms\tremaining: 7.96s\n",
      "81:\tlearn: 33.4712556\ttest: 26.7222602\tbest: 25.1277691 (27)\ttotal: 708ms\tremaining: 7.92s\n",
      "82:\tlearn: 33.4464758\ttest: 26.7353444\tbest: 25.1277691 (27)\ttotal: 710ms\tremaining: 7.84s\n",
      "83:\tlearn: 33.4452780\ttest: 26.7419779\tbest: 25.1277691 (27)\ttotal: 711ms\tremaining: 7.75s\n",
      "84:\tlearn: 33.4442754\ttest: 26.7481467\tbest: 25.1277691 (27)\ttotal: 712ms\tremaining: 7.67s\n",
      "85:\tlearn: 33.4431218\ttest: 26.7552291\tbest: 25.1277691 (27)\ttotal: 714ms\tremaining: 7.59s\n",
      "86:\tlearn: 33.0816775\ttest: 26.8524166\tbest: 25.1277691 (27)\ttotal: 723ms\tremaining: 7.58s\n",
      "87:\tlearn: 33.0806650\ttest: 26.8584684\tbest: 25.1277691 (27)\ttotal: 724ms\tremaining: 7.51s\n",
      "88:\tlearn: 32.8327825\ttest: 26.9279990\tbest: 25.1277691 (27)\ttotal: 730ms\tremaining: 7.47s\n",
      "89:\tlearn: 32.8246174\ttest: 26.9281472\tbest: 25.1277691 (27)\ttotal: 731ms\tremaining: 7.39s\n",
      "90:\tlearn: 32.5783996\ttest: 26.9705118\tbest: 25.1277691 (27)\ttotal: 733ms\tremaining: 7.32s\n",
      "91:\tlearn: 32.5778890\ttest: 26.9730429\tbest: 25.1277691 (27)\ttotal: 735ms\tremaining: 7.25s\n",
      "92:\tlearn: 32.5708489\ttest: 26.9671530\tbest: 25.1277691 (27)\ttotal: 740ms\tremaining: 7.22s\n",
      "93:\tlearn: 32.5545686\ttest: 26.9716454\tbest: 25.1277691 (27)\ttotal: 749ms\tremaining: 7.22s\n",
      "94:\tlearn: 32.5473964\ttest: 26.9717544\tbest: 25.1277691 (27)\ttotal: 751ms\tremaining: 7.15s\n",
      "95:\tlearn: 32.4815056\ttest: 26.9465069\tbest: 25.1277691 (27)\ttotal: 756ms\tremaining: 7.12s\n",
      "96:\tlearn: 32.4740575\ttest: 26.9462317\tbest: 25.1277691 (27)\ttotal: 758ms\tremaining: 7.06s\n",
      "97:\tlearn: 32.4330768\ttest: 26.9468016\tbest: 25.1277691 (27)\ttotal: 761ms\tremaining: 7s\n",
      "98:\tlearn: 32.2025026\ttest: 27.0175424\tbest: 25.1277691 (27)\ttotal: 767ms\tremaining: 6.98s\n",
      "99:\tlearn: 32.2021130\ttest: 27.0197894\tbest: 25.1277691 (27)\ttotal: 768ms\tremaining: 6.91s\n",
      "100:\tlearn: 32.1962611\ttest: 27.0203336\tbest: 25.1277691 (27)\ttotal: 769ms\tremaining: 6.84s\n",
      "101:\tlearn: 32.1246730\ttest: 26.9955897\tbest: 25.1277691 (27)\ttotal: 771ms\tremaining: 6.79s\n",
      "102:\tlearn: 32.0721630\ttest: 27.0011595\tbest: 25.1277691 (27)\ttotal: 789ms\tremaining: 6.87s\n",
      "103:\tlearn: 32.0718833\ttest: 27.0029824\tbest: 25.1277691 (27)\ttotal: 790ms\tremaining: 6.81s\n",
      "104:\tlearn: 32.0667230\ttest: 27.0030461\tbest: 25.1277691 (27)\ttotal: 791ms\tremaining: 6.75s\n",
      "105:\tlearn: 32.0272466\ttest: 27.0052297\tbest: 25.1277691 (27)\ttotal: 797ms\tremaining: 6.72s\n",
      "106:\tlearn: 32.0032111\ttest: 27.0001234\tbest: 25.1277691 (27)\ttotal: 815ms\tremaining: 6.8s\n",
      "107:\tlearn: 32.0029398\ttest: 27.0018532\tbest: 25.1277691 (27)\ttotal: 816ms\tremaining: 6.74s\n",
      "Stopped by overfitting detector  (80 iterations wait)\n",
      "\n",
      "bestTest = 25.12776906\n",
      "bestIteration = 27\n",
      "\n",
      "Shrink model to first 28 iterations.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostRegressor at 0x1b8f5c4ed08>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_final, y_train_final, cat_features=categorical_features_indices,\\\n",
    "          eval_set=(X_validation, y_validation), early_stopping_rounds = 80,\\\n",
    "          use_best_model=True, logging_level = \"Verbose\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With this we can see that the best **RMSE** value of **25.13** (on **validation set**) was acheived at step **27** with no futher improvement after **80** iterations (so the training stopped). We now retain this model as the **best model**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Predictions and Fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 15.2911\n",
      "MAPE: inf\n",
      "RMSE: 29.2967\n",
      "\n",
      "MAPE(replaced zeros): 130.7210 %\n",
      "\n",
      "y_test M and SD (for comparison)\n",
      "31.5728 44.1218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Andrea\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "#Predict on the original Test Set\n",
    "predictions = model.predict(X_test)\n",
    "truevalues = np.array(y_test)\n",
    "\n",
    "#Calculate MAE, MAPE (if no 0 values) and RMSE\n",
    "print(\"MAE:\", '%.4f' % mean_absolute_error(truevalues, predictions))\n",
    "print(\"MAPE:\", '%.4f' % mean_absolute_percentage_error(truevalues, predictions))\n",
    "print(\"RMSE:\", '%.4f' %  np.sqrt(mean_squared_error(truevalues, predictions)))\n",
    "\n",
    "#Replace zero (only for demonstrational purpose) and calculate MAPE\n",
    "truevalues[truevalues == 0] = 1\n",
    "print(\"\\nMAPE(replaced zeros):\", '%.4f' % mean_absolute_percentage_error(truevalues, predictions), \"%\")\n",
    "\n",
    "#Compare with variable, just to get an idea\n",
    "print(\"\\ny_test M and SD (for comparison)\")\n",
    "print('%.4f' % y_test.mean(),'%.4f' % y_test.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Monte Carlo Cross-Validation\n",
    "Now repeat the process 1,000 times and provide average fit statistics, with their standard deviation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save accuracy and kappa scores in a list\n",
    "MAE, RMSE = [], []\n",
    "\n",
    "#For demonstrational purposes we now reapet it 10 times\n",
    "for i in range(0,10):\n",
    "    #Split with no random seed in train, validation and test set\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75)\n",
    "    X_train_final, X_validation, y_train_final, y_validation = train_test_split(X_train, y_train, train_size=0.75)\n",
    "    \n",
    "    model.fit(X_train_final, y_train_final, cat_features=categorical_features_indices, \\\n",
    "              eval_set=(X_validation, y_validation), early_stopping_rounds = 80, use_best_model=True, \\\n",
    "              logging_level = \"Silent\")\n",
    "    \n",
    "    predictions = model.predict(X_test)\n",
    "    truevalues = np.array(y_test)\n",
    "    \n",
    "    MAE.append(mean_absolute_error(truevalues, predictions))\n",
    "    RMSE.append(np.sqrt(mean_squared_error(truevalues, predictions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE at each cross-validation step\n",
      " [15.349770226229547, 12.050911218763286, 15.980185837429588, 18.39281760918325, 14.041911403186912, 14.597220030282347, 13.702341232634197, 13.474771506558714, 13.117398242733515, 12.528981643674657] \n",
      "\n",
      "RMSE at each cross-validation step\n",
      " [28.139670840662156, 29.341372292185817, 38.04193543636353, 60.782859370224884, 31.769903555965612, 43.880098554202554, 38.568208757964, 27.98350059775569, 22.679595556337027, 26.480237694644828] \n",
      "\n",
      "MAE M 14.3236 SD 1.7736 \n",
      "\n",
      "RMSE M 34.7667 SD 10.6216\n"
     ]
    }
   ],
   "source": [
    "print(\"MAE at each cross-validation step\\n\", MAE, \"\\n\")\n",
    "print(\"RMSE at each cross-validation step\\n\", RMSE, \"\\n\")\n",
    "print(\"MAE M\", '%.4f' % np.mean(MAE), \"SD\", '%.4f' % np.std(MAE), \"\\n\")\n",
    "print(\"RMSE M\", '%.4f' % np.mean(RMSE), \"SD\", '%.4f' % np.std(RMSE))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "widgets": {
   "state": {
    "c26d03b66add4e078d26695cab837033": {
     "views": [
      {
       "cell_index": 21
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}