Conv2D - MaxPooling(Q-A).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conv2D - MaxPooling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/inversi/.conda/envs/TextProject/lib/python3.5/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import certifi\n",
    "import unicodedata\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import csv\n",
    "import math\n",
    "import requests\n",
    "import time\n",
    "import random\n",
    "import scipy.stats as stats\n",
    "import matplotlib.pyplot as plt\n",
    "import gensim\n",
    "import keras\n",
    "import h5py\n",
    "import pydot\n",
    "import os\n",
    "import sklearn\n",
    "import operator\n",
    "\n",
    "from keras.layers.merge import Add\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical, plot_model\n",
    "from keras.layers import Conv2D, MaxPooling2D, Embedding, Merge, LSTM, Bidirectional, GlobalMaxPooling1D, Dot, Lambda, Dropout\n",
    "from keras.layers import Average, dot,Permute, Lambda,Layer, add, Concatenate, Dense, LSTM, Input, concatenate, merge, Add, Reshape, Flatten\n",
    "from keras.models import Model, load_model\n",
    "from keras.models import Sequential\n",
    "from numpy import newaxis\n",
    "from keras.models import model_from_json\n",
    "from keras import callbacks\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping,CSVLogger\n",
    "from collections import defaultdict\n",
    "from scipy.stats.stats import pearsonr\n",
    "from keras import backend as K\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from keras import metrics\n",
    "\n",
    "K.os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "K.os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# LOAD WORD2VEC MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 214690 word vectors.\n"
     ]
    }
   ],
   "source": [
    "#LOAD GENSIM MODEL WITH WORD2VEC \n",
    "model_W2C = gensim.models.Word2Vec.load('model/vector_50/mymodelFull')\n",
    "\n",
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "f = open('wiki-w2v/it-vectors.50.5.50.w2v', encoding = \"utf-8\")\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "vocab_size = len(model_W2C.wv.vocab)\n",
    "embedding_matrix = np.zeros((vocab_size, 50))\n",
    "for i in range(len(model_W2C.wv.vocab)):\n",
    "    embedding_vector = embeddings_index.get(model_W2C.wv.index2word[i])\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA TRAINING MANIPULATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOAD TOKENIZER FILES\n",
    "def loadSentences(filename):\n",
    "    array = []\n",
    "    with open(filename, encoding = \"utf-8\") as f:\n",
    "        ar = []\n",
    "        for line in f:\n",
    "            if (str(line) != \"[\\n\") & (str(line) != \"]\\n\"):\n",
    "                ar.append(re.sub('\\n$', '', line))\n",
    "                #print (line)\n",
    "            if (str(line) == \"]\\n\"):\n",
    "                array.append(ar)\n",
    "                ar = []\n",
    "            \n",
    "    f.close()\n",
    "    return array\n",
    "\n",
    "#ANSWER TRAINING\n",
    "original_A_train = loadSentences(\"tokenizer/tokenizerAnswer.txt\")\n",
    "A_train = loadSentences(\"tokenizer/tokenizerAnswer.txt\")\n",
    "\n",
    "#QUESTION  TRAINING\n",
    "original_Q_train = loadSentences(\"tokenizer/tokenizerQuestion.txt\")\n",
    "Q_train = loadSentences(\"tokenizer/tokenizerQuestion.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ANSWER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Il', 'deposito', 'cauzionale', 'e', 'una', 'somma', 'di', 'denaro', 'che', \"l'\", 'utente', 'versa', 'al', 'gestore', 'a', 'titolo', 'di', 'garanzia', 'e', 'che', 'deve', 'essere', 'restituita', 'dopo', 'la', 'cessazione', 'del', 'contratto', 'nel', 'rispetto', 'delle', 'condizioni', 'contrattuali', 'in', 'vigore']\n"
     ]
    }
   ],
   "source": [
    "print(A_train[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "QUESTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Cosa', 'è', 'il', 'deposito', 'cauzionale']\n"
     ]
    }
   ],
   "source": [
    "print(Q_train[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "812\n",
      "812\n"
     ]
    }
   ],
   "source": [
    "#DATASET MANIPULATION FOR TRAINING\n",
    "random_idx = [random.randrange(0, len(original_Q_train)) for i in range(0, len(original_Q_train))]\n",
    "\n",
    "new = []\n",
    "for i in random_idx:#len(A_train)):\n",
    "    new.append(original_Q_train[i])\n",
    "Q_train.extend(new)\n",
    "\n",
    "new = []\n",
    "for i in random_idx:#len(A_train)):\n",
    "    new_random = random.randrange(0, len(random_idx))\n",
    "    while (new_random == i):\n",
    "        new_random = random.randrange(0, len(random_idx))\n",
    "    new.append(original_A_train[new_random])\n",
    "A_train.extend(new)\n",
    "\n",
    "print(len(A_train))\n",
    "print(len(Q_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "812 812 812\n"
     ]
    }
   ],
   "source": [
    "#Y_train LABEL CLASS FOR EVALUATION TRAINING\n",
    "Y_train = []\n",
    "for i in range(0, (int(len(A_train)/2))):\n",
    "    a=[]\n",
    "    a.append(1)\n",
    "    Y_train.append(a)\n",
    "for i in range(int(len(A_train)/2+1), len(A_train) + 1):\n",
    "    a=[]\n",
    "    a.append(0)\n",
    "    Y_train.append(a)\n",
    "Y_train = np.array(Y_train)\n",
    "print(len(Q_train),len(A_train),len(Y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SUBSTITUTE WORD WITH INDEX\n",
    "def replace_matched_items(array, dictionary):\n",
    "       for lst in array:\n",
    "              for ind, item in enumerate(lst):\n",
    "                          lst[ind] = dictionary.get(item, item)       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CLEAN ZEROS\n",
    "def cleaningZero(array):\n",
    "       for elem in array:\n",
    "            while 0 in elem: \n",
    "                elem.remove(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#vocab - uso lo stesso vocab\n",
    "vocab = dict([(k, v.index) for k, v in model_W2C.wv.vocab.items()])\n",
    "\n",
    "#REPLACE A_TRAIN AND Q_TRAIN WITH VOCAB INDEX\n",
    "replace_matched_items(Q_train,vocab)\n",
    "cleaningZero(Q_train)\n",
    "replace_matched_items(A_train,vocab)\n",
    "cleaningZero(A_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PAD DOCUMENTS FOR QUESTION AND ANSWER\n",
    "max_len = max(len(max(A_train,key=len)), len(max(Q_train,key=len)))\n",
    "Q_train_padded = keras.preprocessing.sequence.pad_sequences(Q_train, padding='post', maxlen = max_len)\n",
    "A_train_padded = keras.preprocessing.sequence.pad_sequences(A_train, padding='post', maxlen = max_len)\n",
    "#print(Q_train_padded.shape, A_train_padded.shape,Y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KERAS MODEL (INPUTS: Q - A)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FUNCTON FOR MATRIX MULTIPLICATION\n",
    "class OuterProduct(Layer):\n",
    "    def __init__(self,name=\"OuterProduct\",**kwargs):\n",
    "        super().__init__()\n",
    "\n",
    "    def build(self,input_shape):\n",
    "        self.length = input_shape[0][1]\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self,x):\n",
    "        a, b = x\n",
    "#        print(a.shape,b.shape,a[:, newaxis, :].shape,b[:, :, newaxis].shape)\n",
    "        outerProduct = a[:, newaxis, :] * b[:, :, newaxis]\n",
    " #       print(outerProduct.shape)\n",
    "        outerProduct = K.sum(outerProduct,axis=-1)\n",
    "  #      print(outerProduct.shape)\n",
    "\n",
    "        return outerProduct\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0][0], input_shape[0][1], input_shape[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the model after every epoch. https://keras.io/callbacks/#modelcheckpoint\n",
    "checkpoint = ModelCheckpoint(\"keras_models/Q_A_Conv2D_MaxPool.h5\", monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "#Stop training when a monitored quantity has stopped improving. https://keras.io/callbacks/#earlystopping\n",
    "#stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto')\n",
    "csv_logger = CSVLogger('history/CONV_Q-A/log.csv', append=True, separator=';')\n",
    "\n",
    "callbacks_list = [checkpoint,csv_logger]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Ready...\n"
     ]
    }
   ],
   "source": [
    "#LAYER EMBEDDING-BiLSTM FOR ANSWER\n",
    "A_input = Input(batch_shape=(None,A_train_padded.shape[1],), name='A_input')\n",
    "A = Embedding(len(vocab), output_dim=50, weights=[embedding_matrix],name=\"Embedding_A\",trainable=True)(A_input)\n",
    "A_LSTM = Bidirectional(LSTM(50, dropout=0.5, recurrent_dropout=0.5,return_sequences=True),name=\"LSTM_A\")(A)\n",
    "\n",
    "\n",
    "#LAYER EMBEDDING-BiLSTM FOR QUESTIONS\n",
    "Q_input = Input(batch_shape=(None,Q_train_padded.shape[1],),  name='Q_input')\n",
    "Q = Embedding(len(vocab),output_dim=50,name=\"Embedding_Q\",trainable=True)(Q_input)\n",
    "Q_LSTM = Bidirectional(LSTM(50, dropout=0.5, recurrent_dropout=0.5,return_sequences=True),name=\"LSTM_QA\")(Q)\n",
    "\n",
    "#PRODOTTO DELLE MATRICI EMB\n",
    "M1 = OuterProduct()([A, Q])\n",
    "M1 = Reshape((-1,A_train_padded.shape[1],Q_train_padded.shape[1]),name=\"M1_Embedding\")(M1)\n",
    "M2 = OuterProduct()([A_LSTM, Q_LSTM])\n",
    "M2 = Reshape((-1,A_train_padded.shape[1],Q_train_padded.shape[1]),name=\"M2_LSTM\")(M2)\n",
    "M12 = Concatenate(axis=1,name=\"M1xM2\")([M1, M2])\n",
    "\n",
    "\n",
    "#CONVUTIONAL\n",
    "Convo = Conv2D(8, 3, activation='relu', data_format='channels_first',name=\"Convolutional\")(M12)\n",
    "Convo = Dropout(.5)(Convo)\n",
    "\n",
    "#MAXPOOLING\n",
    "Pool = MaxPooling2D(pool_size=(2, 2), data_format='channels_first',name=\"Max_Pooling\")(Convo)\n",
    "Pool = Dropout(.5)(Pool)\n",
    "Pool = Flatten()(Pool)\n",
    "\n",
    "#MLP\n",
    "dense = Dense(200, activation='relu')(Reshape((-1,))(Pool))\n",
    "dense = Dropout(.5)(dense)\n",
    "dense= Dense(100, activation='relu')(dense)\n",
    "dense = Dropout(.5)(dense)\n",
    "\n",
    "output = Dense(1,activation='sigmoid')(dense)\n",
    "\n",
    "model = Model(inputs=[Q_input,A_input], outputs=output)\n",
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy',metrics.mean_squared_error])\n",
    "\n",
    "#model.summary(line_length=200)\n",
    "print(\"Model Ready...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PLOT PNG KERAS MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PRINT MODEL LAYERS ON FILE.PNG\n",
    "plot_model(model, to_file='image/Q_A_Conv2D_MaxPool.png', show_shapes=True, show_layer_names=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRAIN MODEL Q - A\n",
    "with Stratified K-Fold with 10 splits"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "skf = StratifiedKFold(n_splits=10, shuffle=True)\n",
    "def train_and_evaluate__model(m, q1, a2,y3,q4,a5,y6):\n",
    "    history = model.fit([q1,a2],y3, batch_size=12, epochs=50, validation_data=([q4,a5],y6),callbacks = callbacks_list)\n",
    "    accuracy_history = history.history['acc']\n",
    "    val_accuracy_history = history.history['val_acc']\n",
    "    print (\"Last training accuracy: \" + str(accuracy_history[-1]) + \", last validation accuracy: \" + str(val_accuracy_history[-1]))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "c, r = Y_train.shape\n",
    "Y_train_p = Y_train.reshape(c,)\n",
    "\n",
    "for i, (train, test) in enumerate(skf.split(A_train_padded, Y_train_p)):\n",
    "            print (\"Running Fold\", i+1, \"/\", 10)\n",
    "            train_and_evaluate__model(model, Q_train_padded[train],A_train_padded[train],Y_train[train], Q_train_padded[test],A_train_padded[test],Y_train[test])\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "without strtified K-fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 730 samples, validate on 82 samples\n",
      "Epoch 1/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.8293 - acc: 0.6731 - mean_squared_error: 0.2334Epoch 00000: val_loss did not improve\n",
      "730/730 [==============================] - 132s - loss: 0.8300 - acc: 0.6712 - mean_squared_error: 0.2342 - val_loss: 0.7377 - val_acc: 0.5488 - val_mean_squared_error: 0.2709\n",
      "Epoch 2/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.5824 - acc: 0.7310 - mean_squared_error: 0.1954Epoch 00001: val_loss did not improve\n",
      "730/730 [==============================] - 131s - loss: 0.5822 - acc: 0.7301 - mean_squared_error: 0.1954 - val_loss: 0.9885 - val_acc: 0.4146 - val_mean_squared_error: 0.3667\n",
      "Epoch 3/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.5101 - acc: 0.7628 - mean_squared_error: 0.1695Epoch 00002: val_loss did not improve\n",
      "730/730 [==============================] - 131s - loss: 0.5091 - acc: 0.7644 - mean_squared_error: 0.1691 - val_loss: 1.3296 - val_acc: 0.3780 - val_mean_squared_error: 0.4463\n",
      "Epoch 4/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.4589 - acc: 0.7945 - mean_squared_error: 0.1522Epoch 00003: val_loss did not improve\n",
      "730/730 [==============================] - 131s - loss: 0.4594 - acc: 0.7932 - mean_squared_error: 0.1525 - val_loss: 1.4681 - val_acc: 0.4756 - val_mean_squared_error: 0.4481\n",
      "Epoch 5/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.3957 - acc: 0.8400 - mean_squared_error: 0.1255Epoch 00004: val_loss did not improve\n",
      "730/730 [==============================] - 131s - loss: 0.3938 - acc: 0.8411 - mean_squared_error: 0.1247 - val_loss: 1.8589 - val_acc: 0.4634 - val_mean_squared_error: 0.4640\n",
      "Epoch 6/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.3287 - acc: 0.8703 - mean_squared_error: 0.1013Epoch 00005: val_loss did not improve\n",
      "730/730 [==============================] - 130s - loss: 0.3296 - acc: 0.8685 - mean_squared_error: 0.1018 - val_loss: 2.5735 - val_acc: 0.4024 - val_mean_squared_error: 0.5170\n",
      "Epoch 7/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.2806 - acc: 0.8648 - mean_squared_error: 0.0886Epoch 00006: val_loss did not improve\n",
      "730/730 [==============================] - 131s - loss: 0.2794 - acc: 0.8658 - mean_squared_error: 0.0881 - val_loss: 2.2371 - val_acc: 0.4878 - val_mean_squared_error: 0.4649\n",
      "Epoch 8/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.2372 - acc: 0.8966 - mean_squared_error: 0.0720Epoch 00007: val_loss did not improve\n",
      "730/730 [==============================] - 130s - loss: 0.2453 - acc: 0.8945 - mean_squared_error: 0.0740 - val_loss: 2.6660 - val_acc: 0.4512 - val_mean_squared_error: 0.4891\n",
      "Epoch 9/80\n",
      "725/730 [============================>.] - ETA: 0s - loss: 0.2037 - acc: 0.9214 - mean_squared_error: 0.0619Epoch 00008: val_loss did not improve\n",
      "730/730 [==============================] - 131s - loss: 0.2028 - acc: 0.9219 - mean_squared_error: 0.0615 - val_loss: 2.9108 - val_acc: 0.4268 - val_mean_squared_error: 0.5207\n",
      "Epoch 10/80\n",
      "175/730 [======>.......................] - ETA: 95s - loss: 0.1359 - acc: 0.9429 - mean_squared_error: 0.0395 "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-33b13501ae3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mQ_train_padded\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mA_train_padded\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m80\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcallbacks_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2267\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2268\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2269\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2270\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2271\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1125\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1126\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/TextProject/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1304\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1305\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1308\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit([Q_train_padded,A_train_padded],Y_train, batch_size=25, epochs=80, validation_split=0.1,callbacks = callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# summarize history for accuracy\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "plt.savefig(fname)\n",
    "\n",
    "# summarize history for loss\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "# summarize history for accuracy\n",
    "plt.plot(history.history['mean_squared_error'])\n",
    "plt.plot(history.history['val_mean_squared_error'])\n",
    "plt.title('model mean_squared_error')\n",
    "plt.ylabel('mean_squared_errory')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TextProject",
   "language": "python",
   "name": "textproject"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}