WordEmbeddings_Word2Vec_Softmax.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: numpy in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\n",
      "Requirement already satisfied: scipy in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\n",
      "Requirement already satisfied: numpy>=1.8.2 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from scipy)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib)\n",
      "Requirement already satisfied: python-dateutil>=2.0 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib)\n",
      "Requirement already satisfied: six>=1.10 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib)\n",
      "Requirement already satisfied: numpy>=1.7.1 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib)\n",
      "Requirement already satisfied: pytz in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from matplotlib)\n",
      "Requirement already satisfied: pandas in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\n",
      "Requirement already satisfied: pytz>=2011k in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from pandas)\n",
      "Requirement already satisfied: numpy>=1.9.0 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from pandas)\n",
      "Requirement already satisfied: python-dateutil>=2 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from pandas)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\jenglish\\appdata\\local\\programs\\python\\python36\\lib\\site-packages (from python-dateutil>=2->pandas)\n"
     ]
    }
   ],
   "source": [
    "!pip install numpy\n",
    "!pip install scipy\n",
    "!pip install matplotlib\n",
    "!pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import zipfile\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "DOWNLOADED_FILENAME = 'SampleText.zip'\n",
    "\n",
    "def maybe_download(url_path, expected_bytes):\n",
    "    if not os.path.exists(DOWNLOADED_FILENAME):\n",
    "        filename, _ = urllib.request.urlretrieve(url_path, DOWNLOADED_FILENAME)\n",
    "    statinfo = os.stat(DOWNLOADED_FILENAME)\n",
    "    if statinfo.st_size == expected_bytes:\n",
    "        print('Found and verified file from this path: ',  url_path)\n",
    "        print('Downloaded file: ', DOWNLOADED_FILENAME)\n",
    "    else:\n",
    "        print(statinfo.st_size)\n",
    "        raise Exception('Failed to verify file from: ' + url_path + '. Can you get to it with a browser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_words():\n",
    "    with zipfile.ZipFile(DOWNLOADED_FILENAME) as f:\n",
    "        firstfile = f.namelist()[0]\n",
    "        filestring = tf.compat.as_str(f.read(firstfile))\n",
    "        words = filestring.split()\n",
    "        \n",
    "        return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified file from this path:  http://mattmahoney.net/dc/text8.zip\n",
      "Downloaded file:  SampleText.zip\n"
     ]
    }
   ],
   "source": [
    "URL_PATH = 'http://mattmahoney.net/dc/text8.zip'\n",
    "FILESIZE = 31344016\n",
    "maybe_download(URL_PATH, FILESIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary = read_words()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17005207"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['anarchism',\n",
       " 'originated',\n",
       " 'as',\n",
       " 'a',\n",
       " 'term',\n",
       " 'of',\n",
       " 'abuse',\n",
       " 'first',\n",
       " 'used',\n",
       " 'against',\n",
       " 'early',\n",
       " 'working',\n",
       " 'class',\n",
       " 'radicals',\n",
       " 'including',\n",
       " 'the',\n",
       " 'diggers',\n",
       " 'of',\n",
       " 'the',\n",
       " 'english',\n",
       " 'revolution',\n",
       " 'and',\n",
       " 'the',\n",
       " 'sans',\n",
       " 'culottes']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    word_counts = [['UNKNOWN', -1]]\n",
    "    \n",
    "    counter = collections.Counter(words)\n",
    "    word_counts.extend(counter.most_common(n_words -  1))\n",
    "    \n",
    "    dictionary = dict()\n",
    "    \n",
    "    for word, _ in word_counts:\n",
    "        dictionary[word] = len(dictionary)\n",
    "        \n",
    "    word_indexes = list()\n",
    "        \n",
    "    unknown_count = 0\n",
    "    for word in words:\n",
    "        if word in dictionary:\n",
    "            index = dictionary[word]\n",
    "        else:\n",
    "            index = 0 # dictionary['UNKNOWN']\n",
    "            unknown_count += 1\n",
    "            \n",
    "        word_indexes.append(index)\n",
    "        \n",
    "    word_counts[0][1] = unknown_count\n",
    "    \n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    \n",
    "    return word_counts, word_indexes, dictionary, reversed_dictionary\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCABULARY_SIZE = 5000\n",
    "word_counts, word_indexes, dictionary, reversed_dictionary = build_dataset(vocabulary, VOCABULARY_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['UNKNOWN', 2735459],\n",
       " ('the', 1061396),\n",
       " ('of', 593677),\n",
       " ('and', 416629),\n",
       " ('one', 411764),\n",
       " ('in', 372201),\n",
       " ('a', 325873),\n",
       " ('to', 316376),\n",
       " ('zero', 264975),\n",
       " ('nine', 250430)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 3081,\n",
       " 12,\n",
       " 6,\n",
       " 195,\n",
       " 2,\n",
       " 3134,\n",
       " 46,\n",
       " 59,\n",
       " 156,\n",
       " 128,\n",
       " 742,\n",
       " 477,\n",
       " 0,\n",
       " 134,\n",
       " 1,\n",
       " 0,\n",
       " 2,\n",
       " 1,\n",
       " 103,\n",
       " 855,\n",
       " 3,\n",
       " 1,\n",
       " 0,\n",
       " 0]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_indexes[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "si : 3629\n",
      "julian : 3779\n",
      "college : 354\n",
      "language : 141\n",
      "somewhat : 1355\n",
      "southeast : 3186\n",
      "safe : 3226\n",
      "off : 338\n",
      "vacuum : 3988\n",
      "map : 936\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "for key in random.sample(list(dictionary), 10):\n",
    "    print(key, ':', dictionary[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "587 : female\n",
      "480 : particular\n",
      "4924 : microwave\n",
      "225 : human\n",
      "4529 : dialogue\n",
      "639 : received\n",
      "3050 : driver\n",
      "3898 : grace\n",
      "1117 : ability\n",
      "2296 : regard\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "for key in random.sample(list(reversed_dictionary), 10):\n",
    "    print(key, ':', reversed_dictionary[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "del vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Global index into words maintained across batches\n",
    "global_index = 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(word_indexes, batch_size, num_skips, skip_window):\n",
    "    global global_index\n",
    "\n",
    "    # For every input we find num_skips context words within a window\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    \n",
    "    # batch = [1, 2, 3, .... batch_size]\n",
    "    # labels = [[1], [2], [3], ..., [batch_size]]\n",
    "    batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "    \n",
    "    # The span of a window includes the skip_window elements on each side\n",
    "    # of the input word plus the word itself\n",
    "    span = 2 * skip_window + 1  # [ skip_window input_word skip_window ]\n",
    "\n",
    "    # A deque is double-ended queue which supports memory efficient appends\n",
    "    # and pops from each side\n",
    "    buffer = collections.deque(maxlen=span)\n",
    "\n",
    "    # Initialize the deque with the first words in the deque\n",
    "    for _ in range(span):\n",
    "        buffer.append(word_indexes[global_index])\n",
    "        global_index = (global_index + 1) % len(word_indexes)\n",
    "    \n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window  # input word at the center of the buffer\n",
    "        targets_to_avoid = [skip_window]\n",
    "\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                target = random.randint(0, span - 1)\n",
    "            targets_to_avoid.append(target)\n",
    "            \n",
    "            batch[i * num_skips + j] = buffer[skip_window]  # this is the input word\n",
    "            labels[i * num_skips + j, 0] = buffer[target]  # these are the context words\n",
    "        \n",
    "        # The first word from the buffer is removed automatically when a new word\n",
    "        # is added in at the end\n",
    "        buffer.append(word_indexes[global_index])\n",
    "        global_index = (global_index + 1) % len(word_indexes)\n",
    "    \n",
    "    # Backtrack a little bit to avoid skipping words in the end of a batch, these\n",
    "    # words will be captured in the next batch\n",
    "    global_index = (global_index + len(word_indexes) - span) % len(word_indexes)\n",
    "\n",
    "    return batch, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch, labels = generate_batch(word_indexes, 10, 2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   2,    2, 3134, 3134,   46,   46,   59,   59,  156,  156])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3081],\n",
       "       [3134],\n",
       "       [  59],\n",
       "       [ 156],\n",
       "       [ 195],\n",
       "       [   2],\n",
       "       [ 128],\n",
       "       [  46],\n",
       "       [   0],\n",
       "       [  46]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "of :  originated\n",
      "of :  abuse\n",
      "abuse :  used\n",
      "abuse :  against\n",
      "first :  term\n",
      "first :  of\n",
      "used :  early\n",
      "used :  first\n",
      "against :  UNKNOWN\n"
     ]
    }
   ],
   "source": [
    "for i in range(9):\n",
    "    print(reversed_dictionary[batch[i]], ': ', reversed_dictionary[labels[i][0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_index = 0\n",
    "valid_size = 16\n",
    "valid_window = 100\n",
    "\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "embedding_size = 50\n",
    "skip_window = 2\n",
    "num_skips = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_dataset = tf.constant(valid_examples, dtype=tf.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = tf.Variable(\n",
    "    tf.random_uniform([VOCABULARY_SIZE, embedding_size], -1.0, 1.0))\n",
    "embed = tf.nn.embedding_lookup(embeddings, train_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(5000, 50) dtype=float32_ref>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'embedding_lookup:0' shape=(128, 50) dtype=float32>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = tf.Variable(tf.truncated_normal([VOCABULARY_SIZE, embedding_size], stddev=1.0 / math.sqrt(embedding_size)))\n",
    "biases = tf.Variable(tf.zeros([VOCABULARY_SIZE]))\n",
    "hidden_out = tf.matmul(embed, tf.transpose(weights)) + biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add:0' shape=(128, 5000) dtype=float32>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_one_hot = tf.one_hot(train_labels, VOCABULARY_SIZE)\n",
    "\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=hidden_out, labels=train_one_hot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2_norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "\n",
    "normalized_embeddings = embeddings / l2_norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'embedding_lookup_1:0' shape=(16, 50) dtype=float32>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'truediv:0' shape=(5000, 50) dtype=float32>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_steps = 20001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  0 :  8.61723136902\n",
      "Nearest to more using, murray, balls, marvel, blood, learned, judge, fran,\n",
      "Nearest to were technological, ages, era, design, kennedy, suicide, syrian, rocket,\n",
      "Nearest to than titled, outer, job, forward, hunt, operational, m, save,\n",
      "Nearest to many currently, chapter, ms, identify, particles, describes, thought, below,\n",
      "Nearest to often constantine, monster, amiga, incident, discovery, specialized, empire, senator,\n",
      "Nearest to two received, importance, invaded, come, foot, spend, org, differential,\n",
      "Nearest to seven bills, confidence, real, subjects, santa, before, coverage, mainly,\n",
      "Nearest to and into, chance, monetary, optical, gases, rise, buddhism, grey,\n",
      "Nearest to or user, abstract, usually, binary, seems, angel, membrane, communities,\n",
      "Nearest to will defeat, struggle, diplomatic, colonies, chance, gnu, agave, defensive,\n",
      "Nearest to its medal, entertainment, ownership, park, soil, depends, objective, demographics,\n",
      "Nearest to people mammals, eleven, experience, long, palestinian, projects, historically, devil,\n",
      "Nearest to UNKNOWN creative, unsuccessful, patient, sales, share, tournament, painted, prepared,\n",
      "Nearest to no reportedly, extends, refugees, narrative, told, ocean, boys, generic,\n",
      "Nearest to over framework, worth, scholars, electric, urban, holidays, indiana, smith,\n",
      "Nearest to but ritual, overview, cave, change, hands, answer, me, james,\n",
      "\n",
      "\n",
      "Average loss at step  2000 :  6.93157457316\n",
      "Average loss at step  4000 :  6.15458265376\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as session:\n",
    "    init.run()\n",
    "    \n",
    "    average_loss = 0\n",
    "    for step in xrange(num_steps):\n",
    "        batch_inputs, batch_labels = generate_batch(\n",
    "            word_indexes, batch_size, num_skips, skip_window)\n",
    "        \n",
    "        feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "        \n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        \n",
    "        average_loss += loss_val\n",
    "        \n",
    "        # average loss at every step\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "                \n",
    "            print('Average loss at step ', step, ': ', average_loss)\n",
    "            average_loss = 0\n",
    "        \n",
    "        # note that this is expensive (compute heavy)\n",
    "        if step % 10000 == 0:\n",
    "            sim = similarity.eval()\n",
    "            \n",
    "            for i in xrange(valid_size):\n",
    "                valid_word = reversed_dictionary[valid_examples[i]]\n",
    "                top_k = 8 # number of nearest neighbors\n",
    "                \n",
    "                # argsort sorts from highest to lowest, flip the sign so the largest distance away is the smallest number\n",
    "                nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "                \n",
    "                log_str = 'Nearest to %s' % valid_word\n",
    "                \n",
    "                for k in range(top_k):\n",
    "                    close_word = reversed_dictionary[nearest[k]]\n",
    "                    log_str = '%s %s,' % (log_str, close_word)\n",
    "                print(log_str)\n",
    "                \n",
    "            print('\\n')\n",
    "                    \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}