zz__Test_Predict_VG.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import re\n",
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import Counter, defaultdict, deque\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
    "from sklearn.metrics import roc_curve, roc_auc_score, confusion_matrix, classification_report\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "%matplotlib inline\n",
    "\n",
    "# load English tokenizer, tagger, parser, NER, and word vectors\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "import string\n",
    "punctuations = string.punctuation\n",
    "from  spacy.lang.en.stop_words import STOP_WORDS\n",
    "# Creating a Spacy Parser\n",
    "from spacy.lang.en import English\n",
    "from stemming.porter2 import stem\n",
    "from html import unescape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "vg = pd.read_json('../Amazon_Data/Video_Games_5.json.gz', lines=True, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>overall</th>\n",
       "      <th>verified</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>vote</th>\n",
       "      <th>style</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>10 17, 2015</td>\n",
       "      <td>A1HP7NVNPFMA4N</td>\n",
       "      <td>0700026657</td>\n",
       "      <td>Ambrosia075</td>\n",
       "      <td>This game is a bit hard to get the hang of, bu...</td>\n",
       "      <td>but when you do it's great.</td>\n",
       "      <td>1445040000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>07 27, 2015</td>\n",
       "      <td>A1JGAP0185YJI6</td>\n",
       "      <td>0700026657</td>\n",
       "      <td>travis</td>\n",
       "      <td>I played it a while but it was alright. The st...</td>\n",
       "      <td>But in spite of that it was fun, I liked it</td>\n",
       "      <td>1437955200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>02 23, 2015</td>\n",
       "      <td>A1YJWEXHQBWK2B</td>\n",
       "      <td>0700026657</td>\n",
       "      <td>Vincent G. Mezera</td>\n",
       "      <td>ok game.</td>\n",
       "      <td>Three Stars</td>\n",
       "      <td>1424649600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>02 20, 2015</td>\n",
       "      <td>A2204E1TH211HT</td>\n",
       "      <td>0700026657</td>\n",
       "      <td>Grandma KR</td>\n",
       "      <td>found the game a bit too complicated, not what...</td>\n",
       "      <td>Two Stars</td>\n",
       "      <td>1424390400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>12 25, 2014</td>\n",
       "      <td>A2RF5B5H74JLPE</td>\n",
       "      <td>0700026657</td>\n",
       "      <td>jon</td>\n",
       "      <td>great game, I love it and have played it since...</td>\n",
       "      <td>love this game</td>\n",
       "      <td>1419465600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   overall  verified   reviewTime      reviewerID        asin  \\\n",
       "0        5      True  10 17, 2015  A1HP7NVNPFMA4N  0700026657   \n",
       "1        4     False  07 27, 2015  A1JGAP0185YJI6  0700026657   \n",
       "2        3      True  02 23, 2015  A1YJWEXHQBWK2B  0700026657   \n",
       "3        2      True  02 20, 2015  A2204E1TH211HT  0700026657   \n",
       "4        5      True  12 25, 2014  A2RF5B5H74JLPE  0700026657   \n",
       "\n",
       "        reviewerName                                         reviewText  \\\n",
       "0        Ambrosia075  This game is a bit hard to get the hang of, bu...   \n",
       "1             travis  I played it a while but it was alright. The st...   \n",
       "2  Vincent G. Mezera                                           ok game.   \n",
       "3         Grandma KR  found the game a bit too complicated, not what...   \n",
       "4                jon  great game, I love it and have played it since...   \n",
       "\n",
       "                                       summary  unixReviewTime vote style  \\\n",
       "0                  but when you do it's great.      1445040000  NaN   NaN   \n",
       "1  But in spite of that it was fun, I liked it      1437955200  NaN   NaN   \n",
       "2                                  Three Stars      1424649600  NaN   NaN   \n",
       "3                                    Two Stars      1424390400  NaN   NaN   \n",
       "4                               love this game      1419465600  NaN   NaN   \n",
       "\n",
       "  image  \n",
       "0   NaN  \n",
       "1   NaN  \n",
       "2   NaN  \n",
       "3   NaN  \n",
       "4   NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vg.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vg = vg.loc[:,['overall', 'reviewText']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notes\n",
    "- 'asin' = product ID\n",
    "- 'reviewText' = the review text\n",
    "- 'overall' = the star rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 497577 entries, 0 to 497576\n",
      "Data columns (total 2 columns):\n",
      "overall       497577 non-null int64\n",
      "reviewText    497419 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 7.6+ MB\n"
     ]
    }
   ],
   "source": [
    "vg.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean up nan values and change datatype\n",
    "vg = vg.dropna(how='any')\n",
    "vg.loc[:,'overall'] = vg.overall.astype('int16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 497419 entries, 0 to 497576\n",
      "Data columns (total 3 columns):\n",
      "overall         497419 non-null int16\n",
      "reviewText      497419 non-null object\n",
      "pt_sentiment    497419 non-null int16\n",
      "dtypes: int16(2), object(1)\n",
      "memory usage: 9.5+ MB\n"
     ]
    }
   ],
   "source": [
    "vg.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(497419, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5    299623\n",
       "4     93644\n",
       "3     49140\n",
       "1     30879\n",
       "2     24133\n",
       "Name: overall, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vg.overall.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map the sentiment\n",
    "vg.loc[:,'sentiment'] = vg.overall.map({1: 1, 2: 1, 3: 2, 4: 3, 5: 3}).astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map the sentiment\n",
    "vg.loc[:,'pt_sentiment'] = vg.overall.map({1: 0, 2: 0, 3: 1, \n",
    "                                           4: 1, 5: 1}).astype('int16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 497419 entries, 0 to 497576\n",
      "Data columns (total 3 columns):\n",
      "overall         497419 non-null int16\n",
      "reviewText      497419 non-null object\n",
      "pt_sentiment    497419 non-null int16\n",
      "dtypes: int16(2), object(1)\n",
      "memory usage: 9.5+ MB\n"
     ]
    }
   ],
   "source": [
    "vg.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    442407\n",
       "0     55012\n",
       "Name: pt_sentiment, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vg.pt_sentiment.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data for fasttext\n",
    "train_text, test_text, train_labels, test_labels = train_test_split(vg.reviewText, \n",
    "                                                                    vg.pt_sentiment, \n",
    "                                                                    test_size=0.25,\n",
    "                                                                    random_state=42, \n",
    "                                                                    stratify=vg.pt_sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-24 08:36:46,985 https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/models-v0.4/classy-imdb-en-rnn-cuda%3A0/imdb-v0.4.pt not found in cache, downloading to C:\\Users\\CHUCK~1.TUC\\AppData\\Local\\Temp\\tmpk_ntfis4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 1501979561/1501979561 [22:35<00:00, 1107869.54B/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-24 08:59:23,341 copying C:\\Users\\CHUCK~1.TUC\\AppData\\Local\\Temp\\tmpk_ntfis4 to cache at C:\\Users\\chuck.tucker\\.flair\\models\\imdb-v0.4.pt\n",
      "2020-02-24 08:59:29,437 removing temp file C:\\Users\\CHUCK~1.TUC\\AppData\\Local\\Temp\\tmpk_ntfis4\n",
      "2020-02-24 08:59:29,569 loading file C:\\Users\\chuck.tucker\\.flair\\models\\imdb-v0.4.pt\n"
     ]
    }
   ],
   "source": [
    "from flair.models import TextClassifier\n",
    "classifier = TextClassifier.load('en-sentiment')\n",
    "from flair.data import Sentence\n",
    "\n",
    "scores = defaultdict(float)\n",
    "values = defaultdict(str)\n",
    "i = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "124355"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "124355\n"
     ]
    }
   ],
   "source": [
    "# a list of your sentences\n",
    "# sentences = [Sentence(text) for text in test_text]\n",
    "sentences1 = []\n",
    "sentences2 = []\n",
    "sentences3 = []\n",
    "sentences4 = []\n",
    "sentences5 = []\n",
    "test_text1 = test_text[:25000]\n",
    "test_text2 = test_text[25000:50000]\n",
    "test_text3 = test_text[50000:75000]\n",
    "test_text4 = test_text[75000:100000]\n",
    "test_text5 = test_text[100000:]\n",
    "print(len(test_text1) + len(test_text2) + len(test_text3) + len(test_text4) + len(test_text5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences1 = [Sentence(text) for text in test_text1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create default dicts for predictions\n",
    "from collections import defaultdict\n",
    "\n",
    "scores = defaultdict(float)\n",
    "values = defaultdict(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "[enforce fail at ..\\c10\\core\\CPUAllocator.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 134217728 bytes. Buy new RAM!\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-37899b33bab8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# predict for all sentences\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mclassifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmini_batch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\models\\text_classification_model.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, sentences, mini_batch_size, embedding_storage_mode, multi_class_prob, verbose, use_tokenizer)\u001b[0m\n\u001b[0;32m    222\u001b[0m                     \u001b[1;32mcontinue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 224\u001b[1;33m                 \u001b[0mscores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    225\u001b[0m                 predicted_labels = self._obtain_labels(\n\u001b[0;32m    226\u001b[0m                     \u001b[0mscores\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict_prob\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmulti_class_prob\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\models\\text_classification_model.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, sentences)\u001b[0m\n\u001b[0;32m     97\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mList\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mList\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfloat\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdocument_embeddings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m         text_embedding_list = [\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\embeddings.py\u001b[0m in \u001b[0;36membed\u001b[1;34m(self, sentences)\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0meverything_embedded\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatic_embeddings\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 96\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_add_embeddings_internal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     97\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0msentences\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\embeddings.py\u001b[0m in \u001b[0;36m_add_embeddings_internal\u001b[1;34m(self, sentences)\u001b[0m\n\u001b[0;32m   2926\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2927\u001b[0m         \u001b[1;31m# embed words in the sentence\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2928\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membeddings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2929\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2930\u001b[0m         \u001b[0mlengths\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mList\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtokens\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0msentence\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msentences\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\embeddings.py\u001b[0m in \u001b[0;36membed\u001b[1;34m(self, sentences, static_embeddings)\u001b[0m\n\u001b[0;32m    175\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0membedding\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membeddings\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 177\u001b[1;33m             \u001b[0membedding\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    178\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    179\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\embeddings.py\u001b[0m in \u001b[0;36membed\u001b[1;34m(self, sentences)\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0meverything_embedded\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatic_embeddings\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 96\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_add_embeddings_internal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     97\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0msentences\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\embeddings.py\u001b[0m in \u001b[0;36m_add_embeddings_internal\u001b[1;34m(self, sentences)\u001b[0m\n\u001b[0;32m   2039\u001b[0m             \u001b[1;31m# get hidden states from language model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2040\u001b[0m             all_hidden_states_in_lm = self.lm.get_representation(\n\u001b[1;32m-> 2041\u001b[1;33m                 \u001b[0mtext_sentences\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstart_marker\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend_marker\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchars_per_chunk\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2042\u001b[0m             )\n\u001b[0;32m   2043\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\models\\language_model.py\u001b[0m in \u001b[0;36mget_representation\u001b[1;34m(self, strings, start_marker, end_marker, chars_per_chunk)\u001b[0m\n\u001b[0;32m    153\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbatches\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    154\u001b[0m             \u001b[0mbatch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 155\u001b[1;33m             \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrnn_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    156\u001b[0m             \u001b[0moutput_parts\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrnn_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    157\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\flair\\models\\language_model.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, hidden, ordered_sequence_lengths)\u001b[0m\n\u001b[0;32m     76\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflatten_parameters\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 78\u001b[1;33m         \u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     79\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    531\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 532\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    533\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    557\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    558\u001b[0m             result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,\n\u001b[1;32m--> 559\u001b[1;33m                               self.dropout, self.training, self.bidirectional, self.batch_first)\n\u001b[0m\u001b[0;32m    560\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    561\u001b[0m             result = _VF.lstm(input, batch_sizes, hx, self._flat_weights, self.bias,\n",
      "\u001b[1;31mRuntimeError\u001b[0m: [enforce fail at ..\\c10\\core\\CPUAllocator.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 134217728 bytes. Buy new RAM!\n"
     ]
    }
   ],
   "source": [
    "# predict for all sentences\n",
    "classifier.predict(sentences1, mini_batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check predictions\n",
    "i = 0\n",
    "for sentence in sentences1:\n",
    "        scores[i] = sentence.labels[0].score\n",
    "        values[i] = sentence.labels[0].value\n",
    "        i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictions = pd.DataFrame({'probability': scores, 'prediction': values})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append the other sentencesi results when finished"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}