Activation Function comparison - Sigmoid vs ReLu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_val, y_val) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 28, 28)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import Callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_val.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.astype('float32')/255.\n",
    "X_val = X_val.astype('float32')/255."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One-Hot-Encode labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "n_classes = 10\n",
    "y_train = to_categorical(y_train, n_classes)\n",
    "y_val = to_categorical(y_val, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 10)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Flatten the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = np.reshape(X_train, (60000, 784))\n",
    "X_val = np.reshape(X_val, (10000, 784))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_sigmoid = Sequential()\n",
    "model_sigmoid.add(Dense(700, input_dim=784, activation='sigmoid'))\n",
    "model_sigmoid.add(Dense(350, activation='sigmoid')) \n",
    "model_sigmoid.add(Dense(100, activation='sigmoid')) \n",
    "model_sigmoid.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_sigmoid.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 700)               549500    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 350)               245350    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 100)               35100     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 830,960\n",
      "Trainable params: 830,960\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_sigmoid.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      " - 22s - loss: 2.3216 - acc: 0.1058 - val_loss: 2.3002 - val_acc: 0.1060\n",
      "Epoch 2/10\n",
      " - 17s - loss: 2.2980 - acc: 0.1140 - val_loss: 2.2974 - val_acc: 0.1060\n",
      "Epoch 3/10\n",
      " - 16s - loss: 2.2953 - acc: 0.1145 - val_loss: 2.2948 - val_acc: 0.1060\n",
      "Epoch 4/10\n",
      " - 18s - loss: 2.2927 - acc: 0.1140 - val_loss: 2.2919 - val_acc: 0.1060\n",
      "Epoch 5/10\n",
      " - 17s - loss: 2.2899 - acc: 0.1146 - val_loss: 2.2891 - val_acc: 0.1060\n",
      "Epoch 6/10\n",
      " - 19s - loss: 2.2869 - acc: 0.1172 - val_loss: 2.2862 - val_acc: 0.1060\n",
      "Epoch 7/10\n",
      " - 16s - loss: 2.2838 - acc: 0.1174 - val_loss: 2.2831 - val_acc: 0.1060\n",
      "Epoch 8/10\n",
      " - 17s - loss: 2.2807 - acc: 0.1180 - val_loss: 2.2793 - val_acc: 0.1133\n",
      "Epoch 9/10\n",
      " - 16s - loss: 2.2773 - acc: 0.1247 - val_loss: 2.2759 - val_acc: 0.1060\n",
      "Epoch 10/10\n",
      " - 17s - loss: 2.2738 - acc: 0.1219 - val_loss: 2.2720 - val_acc: 0.1077\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x20ff75c0d30>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 256\n",
    "validation_split = 0.2\n",
    "\n",
    "model_sigmoid.fit(X_train, y_train, epochs=n_epochs, batch_size=batch_size,\n",
    " validation_split=validation_split, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing with ReLu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_relu = Sequential()\n",
    "model_relu.add(Dense(700, input_dim=784, activation='relu'))\n",
    "model_relu.add(Dense(350, activation='relu')) \n",
    "model_relu.add(Dense(100, activation='relu')) \n",
    "model_relu.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_relu.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      " - 18s - loss: 1.6479 - acc: 0.6206 - val_loss: 0.9910 - val_acc: 0.8153\n",
      "Epoch 2/10\n",
      " - 18s - loss: 0.7360 - acc: 0.8374 - val_loss: 0.5322 - val_acc: 0.8722\n",
      "Epoch 3/10\n",
      " - 17s - loss: 0.4891 - acc: 0.8772 - val_loss: 0.4044 - val_acc: 0.8977\n",
      "Epoch 4/10\n",
      " - 17s - loss: 0.3997 - acc: 0.8938 - val_loss: 0.3478 - val_acc: 0.9077\n",
      "Epoch 5/10\n",
      " - 17s - loss: 0.3533 - acc: 0.9028 - val_loss: 0.3157 - val_acc: 0.9139\n",
      "Epoch 6/10\n",
      " - 17s - loss: 0.3236 - acc: 0.9094 - val_loss: 0.2959 - val_acc: 0.9173\n",
      "Epoch 7/10\n",
      " - 18s - loss: 0.3026 - acc: 0.9145 - val_loss: 0.2780 - val_acc: 0.9242\n",
      "Epoch 8/10\n",
      " - 18s - loss: 0.2859 - acc: 0.9190 - val_loss: 0.2652 - val_acc: 0.9255\n",
      "Epoch 9/10\n",
      " - 18s - loss: 0.2722 - acc: 0.9224 - val_loss: 0.2542 - val_acc: 0.9286\n",
      "Epoch 10/10\n",
      " - 18s - loss: 0.2604 - acc: 0.9265 - val_loss: 0.2458 - val_acc: 0.9298\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x20f860c2fd0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 256\n",
    "validation_split = 0.2\n",
    "\n",
    "model_relu.fit(X_train, y_train, epochs=n_epochs, batch_size=batch_size,\n",
    " validation_split=validation_split, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}