sample/nlp-embeddings-sentence-elmo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Remark\n",
    "There is some issue when using Tensorflow Hub in Keras\n",
    "https://github.com/tensorflow/hub/issues/13"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Ingestion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 2031\n",
      "Val: 226\n",
      "Test: 1502\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "categories = ['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med']\n",
    "\n",
    "train_raw_df = fetch_20newsgroups(subset='train', categories=categories)\n",
    "test_raw_df = fetch_20newsgroups(subset='test', categories=categories)\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(np.array(train_raw_df.data), train_raw_df.target, test_size=0.1)\n",
    "x_test = np.array(test_raw_df.data)\n",
    "y_test = test_raw_df.target\n",
    "\n",
    "# x_train = [x_train[:200] for x in x_train]\n",
    "\n",
    "print('Train:', len(x_train))\n",
    "print('Val:', len(x_val))\n",
    "print('Test:', len(x_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%reload_ext  autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfhub_dir = '/data/jupyter/common/model/text/tfhub'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added /data/jupyter/common into sys.path.\n"
     ]
    }
   ],
   "source": [
    "import sys, os\n",
    "def add_aion(curr_path=None):\n",
    "    if curr_path is None:\n",
    "        dir_path = os.getcwd()\n",
    "        target_path = os.path.dirname(dir_path)\n",
    "        if target_path not in sys.path:\n",
    "            print('Added %s into sys.path.' % (target_path))\n",
    "            sys.path.insert(0, target_path)\n",
    "            \n",
    "add_aion()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab Size: 105599\n"
     ]
    }
   ],
   "source": [
    "vocab = set()\n",
    "for sentence in x_train:\n",
    "    tokens = sentence.split(' ')\n",
    "    for token in tokens:\n",
    "        vocab.add(token)\n",
    "\n",
    "vocab_size = len(vocab)\n",
    "print('Vocab Size: %d' % (vocab_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train_words.shape: (2031, 80)\n",
      "x_test_words.shape: (1502, 80)\n"
     ]
    }
   ],
   "source": [
    "max_sentence_length = 80\n",
    "word2Idx = {'<padding>': 0, '<unknown>': 1}\n",
    "idx2word = {0: '<padding>', 1: '<unknown>'}\n",
    "\n",
    "def preprocess(text, word2Idx, idx2word, training=False):\n",
    "    if training:\n",
    "        for sentence in text:\n",
    "            tokens = sentence.split(' ')\n",
    "\n",
    "            for token in tokens:\n",
    "                if token not in word2Idx:\n",
    "                    word2Idx[token] = len(word2Idx)\n",
    "                    idx2word[len(word2Idx)-1] = token\n",
    "\n",
    "\n",
    "    word_vectors = np.zeros((len(text), max_sentence_length))\n",
    "    sentence_vectors = []\n",
    "    \n",
    "    for i, sentence in enumerate(text):\n",
    "        ids = []\n",
    "        words = []\n",
    "        tokens = sentence.split(' ')\n",
    "        for token in tokens:\n",
    "            if token in word2Idx:\n",
    "                ids.append(word2Idx[token])\n",
    "                words.append(token)\n",
    "            else:\n",
    "                ids.append(word2Idx['<unknown>'])\n",
    "                words.append('<unknown>')\n",
    "                \n",
    "            if len(ids) >= max_sentence_length:\n",
    "                break\n",
    "\n",
    "        for i in range(max_sentence_length - len(ids)):\n",
    "            ids.append(word2Idx['<padding>'])\n",
    "            words.append('<padding>')\n",
    "\n",
    "        word_vectors[i] = np.asarray(ids)\n",
    "        sentence_vectors.append(' '.join(words))\n",
    "\n",
    "    sentence_vectors = np.asarray(sentence_vectors)\n",
    "    \n",
    "    return word2Idx, idx2word, word_vectors, sentence_vectors\n",
    "\n",
    "word2Idx, idx2word, x_train_words, x_train_sentences = preprocess(\n",
    "    text=x_train, word2Idx=word2Idx, idx2word=idx2word, training=True)\n",
    "print('x_train_words.shape:', x_train_words.shape)\n",
    "\n",
    "word2Idx, idx2word, x_test_words, x_test_sentences = preprocess(\n",
    "    text=x_test, word2Idx=word2Idx, idx2word=idx2word, training=False)\n",
    "print('x_test_words.shape:', x_test_words.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weighted sum of the 3 layers with word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-10-08 16:47:42.166800. [LOADING] file\n",
      "2018-10-08 16:47:42.965872. [LOADED] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow_hub.module.Module at 0x7fa4062d9128>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from aion.embeddings.elmo import ELMoEmbeddings\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras.layers import Input, Lambda, Dense, Embedding, BatchNormalization, Concatenate, LSTM\n",
    "from keras.models import Model\n",
    "\n",
    "elmo_embs = ELMoEmbeddings(layer='elmo', verbose=20)\n",
    "elmo_embs.load(dest_dir=tfhub_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 256)    27033344    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, None, 1024)   0           input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, None, 1280)   0           embedding_1[0][0]                \n",
      "                                                                 lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, None, 1280)   5120        concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 256)          1573888     batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 4)            1028        lstm_1[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 28,613,380\n",
      "Trainable params: 28,610,820\n",
      "Non-trainable params: 2,560\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/10\n",
      "2031/2031 [==============================] - 252s 124ms/step - loss: 0.8638 - acc: 0.6332\n",
      "Epoch 2/10\n",
      "2031/2031 [==============================] - 255s 126ms/step - loss: 0.4296 - acc: 0.8306\n",
      "Epoch 3/10\n",
      "2031/2031 [==============================] - 252s 124ms/step - loss: 0.1862 - acc: 0.9340\n",
      "Epoch 4/10\n",
      "2031/2031 [==============================] - 252s 124ms/step - loss: 0.0925 - acc: 0.9695\n",
      "Epoch 5/10\n",
      "2031/2031 [==============================] - 255s 126ms/step - loss: 0.0541 - acc: 0.9862\n",
      "Epoch 6/10\n",
      "2031/2031 [==============================] - 256s 126ms/step - loss: 0.0398 - acc: 0.9892\n",
      "Epoch 7/10\n",
      "2031/2031 [==============================] - 255s 126ms/step - loss: 0.0229 - acc: 0.9936\n",
      "Epoch 8/10\n",
      "2031/2031 [==============================] - 254s 125ms/step - loss: 0.0233 - acc: 0.9926\n",
      "Epoch 9/10\n",
      "2031/2031 [==============================] - 255s 125ms/step - loss: 0.0166 - acc: 0.9956\n",
      "Epoch 10/10\n",
      "2031/2031 [==============================] - 254s 125ms/step - loss: 0.0217 - acc: 0.9926\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fa3e830b780>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input Layers\n",
    "word_input_layer = Input(shape=(None, ), dtype='int32')\n",
    "elmo_input_layer = Input(shape=(None, ), dtype=tf.string)\n",
    "\n",
    "# Output Layers\n",
    "word_output_layer = Embedding(\n",
    "    input_dim=vocab_size, output_dim=256)(word_input_layer)\n",
    "elmo_output_layer = Lambda(\n",
    "    elmo_embs.to_keras_layer, \n",
    "    output_shape=(None, 1024))(elmo_input_layer)\n",
    "output_layer = Concatenate()(\n",
    "    [word_output_layer, elmo_output_layer])\n",
    "output_layer = BatchNormalization()(output_layer)\n",
    "output_layer = LSTM(\n",
    "    256, dropout=0.2, recurrent_dropout=0.2)(output_layer)\n",
    "output_layer = Dense(4, activation='sigmoid')(output_layer)\n",
    "\n",
    "# Build Model\n",
    "model = Model(\n",
    "    inputs=[word_input_layer, elmo_input_layer], \n",
    "    outputs=output_layer)\n",
    "model.compile(\n",
    "    loss='sparse_categorical_crossentropy', \n",
    "    optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "model.fit(\n",
    "    [x_train_words, x_train_sentences], y_train,\n",
    "#     validation_data=([x_test_words, x_test_sentences], y_test), \n",
    "    epochs=10, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict([x_test_words, x_test_sentences])\n",
    "y_pred = np.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:77.10%\n",
      "Classification Report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.72      0.45      0.55       319\n",
      "          1       0.87      0.85      0.86       389\n",
      "          2       0.88      0.83      0.85       396\n",
      "          3       0.65      0.89      0.75       398\n",
      "\n",
      "avg / total       0.78      0.77      0.76      1502\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print('Accuracy:%.2f%%' % (accuracy_score(y_test, y_pred)*100))\n",
    "print('Classification Report:')\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weighted sum of the 3 layers without word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-10-08 17:32:55.478302. [LOADING] file\n",
      "2018-10-08 17:32:56.379230. [LOADED] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow_hub.module.Module at 0x7fa404373d30>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from aion.embeddings.elmo import ELMoEmbeddings\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras.layers import Input, Lambda, Dense, Embedding, BatchNormalization, Concatenate, LSTM\n",
    "from keras.models import Model\n",
    "\n",
    "elmo_embs = ELMoEmbeddings(layer='elmo', verbose=20)\n",
    "elmo_embs.load(dest_dir=tfhub_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "lambda_2 (Lambda)            (None, None, 1024)        0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, None, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 256)               1311744   \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4)                 1028      \n",
      "=================================================================\n",
      "Total params: 1,316,868\n",
      "Trainable params: 1,314,820\n",
      "Non-trainable params: 2,048\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "2031/2031 [==============================] - 243s 120ms/step - loss: 0.8790 - acc: 0.6140\n",
      "Epoch 2/10\n",
      "2031/2031 [==============================] - 241s 119ms/step - loss: 0.4587 - acc: 0.8159\n",
      "Epoch 3/10\n",
      "2031/2031 [==============================] - 241s 119ms/step - loss: 0.2837 - acc: 0.8887\n",
      "Epoch 4/10\n",
      "2031/2031 [==============================] - 242s 119ms/step - loss: 0.1922 - acc: 0.9311\n",
      "Epoch 5/10\n",
      "2031/2031 [==============================] - 244s 120ms/step - loss: 0.1184 - acc: 0.9641\n",
      "Epoch 6/10\n",
      "2031/2031 [==============================] - 236s 116ms/step - loss: 0.0592 - acc: 0.9852\n",
      "Epoch 7/10\n",
      "2031/2031 [==============================] - 239s 118ms/step - loss: 0.0437 - acc: 0.9902\n",
      "Epoch 8/10\n",
      "2031/2031 [==============================] - 237s 117ms/step - loss: 0.0496 - acc: 0.9882\n",
      "Epoch 9/10\n",
      "2031/2031 [==============================] - 235s 116ms/step - loss: 0.0330 - acc: 0.9931\n",
      "Epoch 10/10\n",
      "2031/2031 [==============================] - 233s 115ms/step - loss: 0.0247 - acc: 0.9936\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fa17bff1940>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input Layers\n",
    "elmo_input_layer = Input(shape=(None, ), dtype=tf.string)\n",
    "\n",
    "# Output Layers\n",
    "output_layer = Lambda(\n",
    "    elmo_embs.to_keras_layer, \n",
    "    output_shape=(None, 1024))(elmo_input_layer)\n",
    "output_layer = BatchNormalization()(output_layer)\n",
    "output_layer = LSTM(\n",
    "    256, dropout=0.2, recurrent_dropout=0.2)(output_layer)\n",
    "output_layer = Dense(4, activation='sigmoid')(output_layer)\n",
    "\n",
    "# Build Model\n",
    "model = Model(\n",
    "    inputs=elmo_input_layer, outputs=output_layer)\n",
    "model.compile(\n",
    "    loss='sparse_categorical_crossentropy', \n",
    "    optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "model.fit(\n",
    "    x_train_sentences, y_train,\n",
    "#     validation_data=([x_test_words, x_test_sentences], y_test), \n",
    "    epochs=10, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(x_test_sentences)\n",
    "y_pred = np.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:76.83%\n",
      "Classification Report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.62      0.59      0.60       319\n",
      "          1       0.88      0.84      0.86       389\n",
      "          2       0.92      0.80      0.86       396\n",
      "          3       0.66      0.81      0.73       398\n",
      "\n",
      "avg / total       0.78      0.77      0.77      1502\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print('Accuracy:%.2f%%' % (accuracy_score(y_test, y_pred)*100))\n",
    "print('Classification Report:')\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fixed Mean-pooling without word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-10-08 18:25:28.769929. [LOADING] file\n",
      "2018-10-08 18:25:29.650498. [LOADED] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow_hub.module.Module at 0x7fa17b5992e8>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from aion.embeddings.elmo import ELMoEmbeddings\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras.layers import Input, Lambda, Dense, Embedding, BatchNormalization, Concatenate, LSTM\n",
    "from keras.models import Model\n",
    "\n",
    "elmo_embs = ELMoEmbeddings(layer='default', verbose=20)\n",
    "elmo_embs.load(dest_dir=tfhub_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "lambda_3 (Lambda)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               262400    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 4)                 1028      \n",
      "=================================================================\n",
      "Total params: 263,428\n",
      "Trainable params: 263,428\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "2031/2031 [==============================] - 224s 110ms/step - loss: 0.6419 - acc: 0.7395\n",
      "Epoch 2/10\n",
      "2031/2031 [==============================] - 225s 111ms/step - loss: 0.3843 - acc: 0.8454\n",
      "Epoch 3/10\n",
      "2031/2031 [==============================] - 226s 111ms/step - loss: 0.3123 - acc: 0.8740\n",
      "Epoch 4/10\n",
      "2031/2031 [==============================] - 231s 114ms/step - loss: 0.2745 - acc: 0.8927\n",
      "Epoch 5/10\n",
      "2031/2031 [==============================] - 225s 111ms/step - loss: 0.2423 - acc: 0.9178\n",
      "Epoch 6/10\n",
      "2031/2031 [==============================] - 223s 110ms/step - loss: 0.2187 - acc: 0.9212\n",
      "Epoch 7/10\n",
      "2031/2031 [==============================] - 224s 110ms/step - loss: 0.2121 - acc: 0.9207\n",
      "Epoch 8/10\n",
      "2031/2031 [==============================] - 221s 109ms/step - loss: 0.1884 - acc: 0.9306\n",
      "Epoch 9/10\n",
      "2031/2031 [==============================] - 221s 109ms/step - loss: 0.1640 - acc: 0.9444\n",
      "Epoch 10/10\n",
      "2031/2031 [==============================] - 224s 110ms/step - loss: 0.1631 - acc: 0.9419\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fa2e9468fd0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Input Layers\n",
    "input_layer = Input(shape=(None,), dtype=tf.string)\n",
    "\n",
    "# Output Layers\n",
    "output_layer = Lambda(\n",
    "    elmo_embs.to_keras_layer, \n",
    "    output_shape=(1024,))(input_layer)\n",
    "output_layer = Dense(\n",
    "    256, activation='relu')(output_layer)\n",
    "output_layer = Dense(4, activation='sigmoid')(output_layer)\n",
    "\n",
    "model = Model(inputs=[input_layer], outputs=output_layer)\n",
    "model.compile(\n",
    "    loss='sparse_categorical_crossentropy', \n",
    "    optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()\n",
    "model.fit(\n",
    "    x_train_sentences, y_train,\n",
    "#     validation_data=([x_test_words, x_test_sentences], y_test), \n",
    "    epochs=10, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(x_test_sentences)\n",
    "y_pred = np.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:74.37%\n",
      "Classification Report:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.71      0.66      0.68       319\n",
      "          1       0.95      0.66      0.78       389\n",
      "          2       0.92      0.74      0.83       396\n",
      "          3       0.57      0.89      0.70       398\n",
      "\n",
      "avg / total       0.79      0.74      0.75      1502\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print('Accuracy:%.2f%%' % (accuracy_score(y_test, y_pred)*100))\n",
    "print('Classification Report:')\n",
    "print(classification_report(y_test, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}