SentimentClassification_NaiveBayes_IMDB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\n",
      "Requirement already satisfied: six in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from nltk)\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk\n",
    "\n",
    "import numpy as np\n",
    "import nltk\n",
    "import re\n",
    "import os\n",
    "import tarfile\n",
    "from six.moves import urllib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.13.3\n",
      "3.2.5\n"
     ]
    }
   ],
   "source": [
    "print(np.__version__)\n",
    "print(nltk.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "DOWNLOADED_FILENAME = 'ImdbReviews.tar.gz'\n",
    "\n",
    "def download_file(url_path):\n",
    "    if not os.path.exists(DOWNLOADED_FILENAME):\n",
    "        filename, _ = urllib.request.urlretrieve(url_path, DOWNLOADED_FILENAME)\n",
    "    print('Found and verified file from this path', url_path)\n",
    "    print('Downloaded file: ', DOWNLOADED_FILENAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "TOKEN_REGEX = re.compile(\"[^A-Za-z0-9 ]+\")\n",
    "\n",
    "def get_reviews(dirname, positive=True):\n",
    "    label = 1 if positive else 0\n",
    "    \n",
    "    reviews = []\n",
    "    \n",
    "    for filename in os.listdir(dirname):\n",
    "        if filename.endswith('.txt'):\n",
    "            \n",
    "            with open(dirname + filename, 'r+') as f:\n",
    "                review = f.read().decode('utf-8')\n",
    "                review = review.lower().replace('<br />', ' ')\n",
    "                review = re.sub(TOKEN_REGEX, '', review)\n",
    "                \n",
    "                # Return a tuple of the review text and a label for whether it is a postive or negative review\n",
    "                reviews.append((review, label))\n",
    "    return reviews\n",
    "\n",
    "def extract_reviews():\n",
    "    \n",
    "    # If the file has not already been extracted\n",
    "    if not os.path.exists('aclImdb'):\n",
    "        with tarfile.open(DOWNLOADED_FILENAME) as tar:\n",
    "            tar.extractall()\n",
    "            tar.close()\n",
    "    positive_reviews = get_reviews('aclImdb/train/pos/', positive=True)\n",
    "    negative_reviews = get_reviews('aclImdb/train/neg/', positive=False)\n",
    "    \n",
    "    return positive_reviews, negative_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified file from this path http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
      "Downloaded file:  ImdbReviews.tar.gz\n"
     ]
    }
   ],
   "source": [
    "URL_PATH = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'\n",
    "download_file(URL_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_reviews, negative_reviews = extract_reviews();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(positive_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(negative_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "positive_reviews[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DATA = 5000\n",
    "TOTAL_DATA = len(positive_reviews)\n",
    "\n",
    "train_reviews = positive_reviews[:TRAIN_DATA] + negative_reviews[:TRAIN_DATA]\n",
    "\n",
    "test_positive_reviews = positive_reviews[TRAIN_DATA:TOTAL_DATA]\n",
    "test_negative_reviews = negative_reviews[TRAIN_DATA:TOTAL_DATA]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vocabulary(train_reviews):\n",
    "    words_set = set()\n",
    "    \n",
    "    for review in train_reviews:\n",
    "        words_set.update(review[0].split())\n",
    "    return list(words_set)\n",
    "\n",
    "vocabulary = get_vocabulary(train_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_features(review_text):\n",
    "    # split the review into words, and create a set of the words\n",
    "    review_words = set(review_text.split())\n",
    "    \n",
    "    features = {}\n",
    "    for word in vocabulary:\n",
    "        features[word] = (word in review_words)\n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = nltk.classify.apply_features(extract_features, train_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_classifier = nltk.NaiveBayesClassifier.train(train_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentiment_calculator(review_text):\n",
    "    features = extract_features(review_text)\n",
    "    return trained_classifier.classify(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_calculatorculator(\"What an amazing movie!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_calculator(\"What a terrible movie\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify_test_reviews(test_positive_reviews, test_negative_reviews, sentiment_calculator):\n",
    "    \n",
    "    positive_results = [sentiment_calculator(review[0]) for review in test_positive_reviews]\n",
    "    negative_results = [sentiment_calculator(review[0]) for review in test_negative_reviews]\n",
    "    \n",
    "    true_positives = sum(x > 0 for x in positive_results)\n",
    "    true_negatives = sum(x == 0 for x in negative_results)\n",
    "    \n",
    "    percent_true_positive = float(true_positives) / len(positive_results)\n",
    "    precent_true_negative = float(true_negatives) / len(negative_results)\n",
    "    \n",
    "    total_accurate = true_positives + true_negatives\n",
    "    total = len(positive_results) + len(negative_results)\n",
    "    \n",
    "    print(\"Accuracy on positive reviews = \" + \"%.2f\" % (percent_true_positive * 100) + \"%\")\n",
    "    print(\"Accuracy on negative reviews = \" + \"%.2f\" % (percent_true_negative * 100) + \"%\")\n",
    "    print(\"Overall Accuracy = \" + \"%.2f\" % (total_accurate * 100 / total) + \"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classify_test_reviews(test_positive_reviews, test_negative_reviews, sentiment_calculator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}