BLSTM_10,5_neurons_3FC_256_dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "context = 5\n",
    "feature_dimension =39\n",
    "fc_neurons = 256\n",
    "tf.set_random_seed(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor 'unstack:0' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:1' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:2' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:3' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:4' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:5' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:6' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:7' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:8' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:9' shape=(?, 39) dtype=float32>, <tf.Tensor 'unstack:10' shape=(?, 39) dtype=float32>]\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder( shape = [None, feature_dimension, 2*context + 1], dtype = tf.float32)\n",
    "y = tf.placeholder( shape = [None, 2], dtype = tf.float32)\n",
    "keep_prob = tf.placeholder(dtype= tf.float32)\n",
    "\n",
    "input_lstm = tf.unstack( x, 2*context +1, 2)\n",
    "print(input_lstm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\genericpath.py\u001b[0m in \u001b[0;36mexists\u001b[1;34m(path)\u001b[0m\n\u001b[0;32m     18\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m         \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     20\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOSError\u001b[0m: [WinError 123] The filename, directory name, or volume label syntax is incorrect: '<frozen importlib._bootstrap>'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-2c20ecab8be8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m lstm_fw = tf.contrib.rnn.LSTMCell( 10, use_peepholes = True,\n\u001b[0m\u001b[0;32m      2\u001b[0m                     initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m lstm_back = tf.contrib.rnn.LSTMCell( 5, use_peepholes = True,\n\u001b[0;32m      5\u001b[0m                     initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\python\\util\\lazy_loader.py\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     52\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 53\u001b[1;33m     \u001b[0mmodule\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_load\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     54\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\python\\util\\lazy_loader.py\u001b[0m in \u001b[0;36m_load\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     40\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_load\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m     \u001b[1;31m# Import the target module and insert it into the parent's namespace\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 42\u001b[1;33m     \u001b[0mmodule\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     43\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent_module_globals\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_local_name\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     44\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\__init__.py\u001b[0m in \u001b[0;36mimport_module\u001b[1;34m(name, package)\u001b[0m\n\u001b[0;32m    124\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    125\u001b[0m             \u001b[0mlevel\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 126\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_bootstrap\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_gcd_import\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlevel\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpackage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    127\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    128\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\_bootstrap.py\u001b[0m in \u001b[0;36m_gcd_import\u001b[1;34m(name, package, level)\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load\u001b[1;34m(name, import_)\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load_unlocked\u001b[1;34m(name, import_)\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\_bootstrap.py\u001b[0m in \u001b[0;36m_load_unlocked\u001b[1;34m(spec)\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\_bootstrap_external.py\u001b[0m in \u001b[0;36mexec_module\u001b[1;34m(self, module)\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\importlib\\_bootstrap.py\u001b[0m in \u001b[0;36m_call_with_frames_removed\u001b[1;34m(f, *args, **kwds)\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcopy_graph\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcrf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 27\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcudnn_rnn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     28\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdeprecated\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\cudnn_rnn\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 28\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCudnnGRU\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     29\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCudnnLSTM\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCudnnRNNRelu\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\cudnn_rnn\\python\\ops\\cudnn_rnn_ops.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcudnn_rnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mgen_cudnn_rnn_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlstm_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     22\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mloader\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcommon_shapes\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\rnn\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     79\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlstm_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 81\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn_cell\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     82\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mstatic_bidirectional_rnn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\rnn\\python\\ops\\rnn_cell.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompiler\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mjit\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlayers\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdtypes\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mop_def_registry\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\layers\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[1;31m# pylint: disable=unused-import,wildcard-import\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 102\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    103\u001b[0m \u001b[1;31m# pylint: enable=unused-import,wildcard-import\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membedding_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoders\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_column\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_column_ops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minitializers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\feature_column.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    136\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mframework\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mvariables\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mcontrib_variables\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    137\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0membedding_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 138\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlayers\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    139\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mbucketization_op\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mops\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msparse_feature_cross_op\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\layers.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m   1377\u001b[0m             \u001b[0mis_training\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1378\u001b[0m             \u001b[0moutputs_collections\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1379\u001b[1;33m             scope=None):\n\u001b[0m\u001b[0;32m   1380\u001b[0m   \"\"\"Returns a dropout op applied to the input.\n\u001b[0;32m   1381\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\contrib\\framework\\python\\ops\\arg_scope.py\u001b[0m in \u001b[0;36madd_arg_scope\u001b[1;34m(func)\u001b[0m\n\u001b[0;32m    182\u001b[0m   \u001b[0m_add_op\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    183\u001b[0m   \u001b[0msetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc_with_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'_key_op'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_key_op\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 184\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0mtf_decorator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_decorator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc_with_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    186\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\site-packages\\tensorflow\\python\\util\\tf_decorator.py\u001b[0m in \u001b[0;36mmake_decorator\u001b[1;34m(target, decorator_func, decorator_name, decorator_doc, decorator_argspec)\u001b[0m\n\u001b[0;32m     84\u001b[0m   \"\"\"\n\u001b[0;32m     85\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0mdecorator_name\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m     \u001b[0mdecorator_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_inspect\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m]\u001b[0m  \u001b[1;31m# Caller's name.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     87\u001b[0m   decorator = TFDecorator(decorator_name, target, decorator_doc,\n\u001b[0;32m     88\u001b[0m                           decorator_argspec)\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\inspect.py\u001b[0m in \u001b[0;36mstack\u001b[1;34m(context)\u001b[0m\n\u001b[0;32m   1489\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1490\u001b[0m     \u001b[1;34m\"\"\"Return a list of records for the stack above the caller's frame.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1491\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgetouterframes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1492\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1493\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mtrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\inspect.py\u001b[0m in \u001b[0;36mgetouterframes\u001b[1;34m(frame, context)\u001b[0m\n\u001b[0;32m   1466\u001b[0m     \u001b[0mframelist\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1467\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[0mframe\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1468\u001b[1;33m         \u001b[0mframeinfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mgetframeinfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1469\u001b[0m         \u001b[0mframelist\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mFrameInfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mframeinfo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1470\u001b[0m         \u001b[0mframe\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mframe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf_back\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\inspect.py\u001b[0m in \u001b[0;36mgetframeinfo\u001b[1;34m(frame, context)\u001b[0m\n\u001b[0;32m   1436\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'{!r} is not a frame or traceback object'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1437\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1438\u001b[1;33m     \u001b[0mfilename\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetsourcefile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mgetfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1439\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcontext\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1440\u001b[0m         \u001b[0mstart\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlineno\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m//\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\inspect.py\u001b[0m in \u001b[0;36mgetsourcefile\u001b[1;34m(object)\u001b[0m\n\u001b[0;32m    688\u001b[0m                  importlib.machinery.EXTENSION_SUFFIXES):\n\u001b[0;32m    689\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 690\u001b[1;33m     \u001b[1;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    691\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    692\u001b[0m     \u001b[1;31m# only return a non-existent filename if the module has a PEP 302 loader\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\WinPython-64bit-3.6.3.0Qt5\\python-3.6.3.amd64\\lib\\genericpath.py\u001b[0m in \u001b[0;36mexists\u001b[1;34m(path)\u001b[0m\n\u001b[0;32m     17\u001b[0m     \u001b[1;34m\"\"\"Test whether a path exists.  Returns False for broken symbolic links\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m         \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     20\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "lstm_fw = tf.contrib.rnn.LSTMCell( 10, use_peepholes = True,\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "\n",
    "lstm_back = tf.contrib.rnn.LSTMCell( 5, use_peepholes = True,\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "\n",
    "blstm_out, state_fw, state_bw = tf.contrib.rnn.stack_bidirectional_rnn( [lstm_fw], [lstm_back], input_lstm, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights1 = tf.get_variable(name='dw1', shape = [15, fc_neurons],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "biases1 = tf.get_variable(name='db1', shape=[fc_neurons],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "\n",
    "dense1 = tf.matmul( blstm_out[-1], weights1) + biases1\n",
    "dense1_out = tf.nn.relu(dense1)\n",
    "dense1_dropout = tf.nn.dropout( dense1_out, keep_prob )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights2 = tf.get_variable(name='dw2', shape = [fc_neurons, fc_neurons],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "biases2 = tf.get_variable(name='db2', shape=[fc_neurons],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "\n",
    "dense2 = tf.matmul( dense1_dropout, weights2) + biases2\n",
    "dense2_out = tf.nn.relu(dense2)\n",
    "dense2_dropout = tf.nn.dropout( dense2_out, keep_prob )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights3 = tf.get_variable(name='dw3', shape = [fc_neurons, fc_neurons],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "biases3 = tf.get_variable(name='db3', shape=[fc_neurons],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "\n",
    "dense3 = tf.matmul( dense2_dropout, weights3) + biases3\n",
    "dense3_out = tf.nn.relu(dense3)\n",
    "dense3_dropout = tf.nn.dropout( dense3_out, keep_prob )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_final = tf.get_variable(name='dfw', shape = [fc_neurons, 2],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "biases_final = tf.get_variable(name='dfb', shape=[2],\n",
    "                    initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32))\n",
    "\n",
    "output = tf.matmul( dense3_dropout, weights_final) + biases_final\n",
    "output1 = tf.nn.softmax(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels = y, logits = output))\n",
    "optimizer = tf.train.AdamOptimizer(1e-4)\n",
    "train_step = optimizer.minimize(cross_entropy)\n",
    "\n",
    "correct_predictions = tf.equal( tf.argmax(output, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean( tf.cast( correct_predictions, tf.float32 ))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# matric for testing on development \n",
    "\n",
    "test_features = []\n",
    "test_labels = []\n",
    "\n",
    "test_path = \"K:\\\\cqcc_database\\\\context_5\\\\CNN_context_dev\"\n",
    "\n",
    "for folders in os.listdir(test_path):\n",
    "    print(folders)\n",
    "    features = np.load(os.path.join(test_path, folders, \"feature_image.npy\"))\n",
    "    print(features.shape)\n",
    "    labels = np.load(os.path.join(test_path, folders, \"label_image.npy\"))\n",
    "    for i in range(labels.shape[0]):\n",
    "        test_features.append(features[i])\n",
    "        test_labels.append(labels[i])\n",
    "    \n",
    "test_features = np.array(test_features)\n",
    "test_labels = np.array(test_labels)\n",
    "print(test_features.shape)\n",
    "print(test_labels.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#initializing tensors using sessoion\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver(max_to_keep=None)\n",
    "features_path = \"K:\\\\MFCC\\\\context_5\\\\CNN_train_data\"\n",
    "development_path = \"K:\\\\MFCC\\\\context_5\\\\CNN_test_data\"\n",
    "# evaluation_path = \"H:\\\\LP_residual_phase\\\\CNN_input_testing_eval\"\n",
    "model_path = \"K:\\\\HT-IACC_BLSTM_120\\\\context_5\\\\BLSTM_10,5_neurons_3FC_512_N\\\\Models\\\\\"\n",
    "    \n",
    "dev_sp_path = \"K:\\\\HT-IACC_BLSTM_120\\\\context_5\\\\BLSTM_10,5_neurons_3FC_512_N\\\\dev\"\n",
    "# eva_sp_path = \"G:\\\\cqcc_database\\\\context_10\\\\BLSTM\\\\eval\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_eer(scores_true, scores_false):\n",
    "    \n",
    "    \"\"\" Calculate EER\n",
    "        Args:\n",
    "            scores_true: A score list of positive samples\n",
    "            scores_false: A score list of negative samples\n",
    "        Return:\n",
    "            (EER, threshold)\n",
    "        Note:\n",
    "            Here, the threshold is set, and if the score is higher than or equal to the threshold,\n",
    "            the decision is passed, and the decision is rejected as below.\n",
    "    \"\"\"\n",
    "    min1 = min(scores_true)\n",
    "    min2 = min(scores_false)\n",
    "    low = min(min1, min2)\n",
    "    max1 = max(scores_true)\n",
    "    max2 = max(scores_false)\n",
    "    high = max(max1, max2)\n",
    "    FAR = -55\n",
    "    FRR = -55\n",
    "    mid = -100\n",
    "    # Bisection find threshold\n",
    "    while True:\n",
    "        bef_FAR = FAR \n",
    "        bef_FRR = FRR\n",
    "        bef_mid = mid\n",
    "        mid = (high + low) / 2\n",
    "        FRR = sum([1 for s in scores_true if s < mid]) / len(scores_true)\n",
    "        FAR = sum([1 for s in scores_false if s >= mid]) / len(scores_false)\n",
    "        #print(\"FRR  \",FRR,\" FAR   \",FAR,\"   mid   \",mid)\n",
    "\n",
    "        if abs(FRR - FAR) < 1e-7:\n",
    "            print(\"FRR  \",FRR,\" FAR   \",FAR,\"   mid   \",mid)\n",
    "            break\n",
    "\n",
    "        if FRR < FAR:\n",
    "            low = mid\n",
    "        else:\n",
    "            high = mid\n",
    "            \n",
    "        if bef_FAR == FAR :\n",
    "            if bef_FRR == FRR :\n",
    "                if bef_mid == mid :\n",
    "                    print(\"FRR  \",FRR,\" FAR   \",FAR,\"   mid   \",mid)\n",
    "                    break\n",
    "\n",
    "    EER  =  FRR\n",
    "    return EER, (high + low) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.io import savemat\n",
    "\n",
    "def get_EER(test_path, index, sp_path ):\n",
    "    scores = []\n",
    "    development_labels = []\n",
    "    probabilities = []\n",
    "\n",
    "    #test_path = \"C:\\\\Users\\\\project1\\\\Desktop\\\\Siva\\\\HT_IFCC_hemlata\\\\CNN_input_testing_dev\"\n",
    "\n",
    "    for folders in os.listdir(test_path):\n",
    "        features = np.load(os.path.join(test_path, folders, \"feature_image.npy\"))\n",
    "        labels = np.load(os.path.join(test_path, folders, \"label_image.npy\"))\n",
    "        patches = labels.shape[0]\n",
    "        patch_prob = session.run(output1, feed_dict = {x:features, keep_prob:0.5})\n",
    "        final_prob = np.zeros([2])\n",
    "        \n",
    "        for i in range(patches):\n",
    "            if(patch_prob[i][0] == 0):\n",
    "                patch_prob[i][0] = 1e-25\n",
    "            if(patch_prob[i][1] == 0):\n",
    "                patch_prob[i][1] = 1e-25\n",
    "                \n",
    "            final_prob[0] = final_prob[0] + np.log(patch_prob[i][0])\n",
    "            final_prob[1] = final_prob[1] + np.log(patch_prob[i][1])\n",
    "\n",
    "        final_prob = final_prob / patches\n",
    "        score = (final_prob[0]) - (final_prob[1])\n",
    "        audio_label = labels[0]\n",
    "\n",
    "        scores.append(score)\n",
    "        probabilities.append( np.mean(patch_prob, axis=0) )\n",
    "        development_labels.append( audio_label )\n",
    "\n",
    "    scores = np.array(scores)\n",
    "    development_labels = np.array(development_labels)\n",
    "    probabilities = np.array(probabilities)\n",
    "\n",
    "    scores_geniue = []\n",
    "    scores_spoof = []\n",
    "    prob_genuine = []\n",
    "    prob_spoof = []\n",
    "\n",
    "    for i in range(development_labels.shape[0]):\n",
    "        if development_labels[i][0] == 1 :\n",
    "            scores_geniue.append(scores[i])\n",
    "            prob_genuine.append(probabilities[i])\n",
    "        elif development_labels[i][1] == 1:\n",
    "            scores_spoof.append(scores[i])\n",
    "            prob_spoof.append(probabilities[i])\n",
    "\n",
    "    scores_geniue = np.array(scores_geniue)\n",
    "    scores_spoof = np.array(scores_spoof)\n",
    "    prob_genuine = np.array(prob_genuine)\n",
    "    prob_spoof = np.array(prob_spoof)\n",
    "    \n",
    "    a,b = cal_eer(scores_geniue, scores_spoof)\n",
    "    \n",
    "    score_path = os.path.join(sp_path, \"scores\")\n",
    "    prob_path = os.path.join(sp_path, \"probabilities\")\n",
    "    \n",
    "    if not os.path.exists(score_path):\n",
    "        os.makedirs(score_path)\n",
    "    if not os.path.exists(prob_path):\n",
    "        os.makedirs(prob_path)\n",
    "        \n",
    "    prob_file = \"probabilities_\"+str(index)\n",
    "    sc_file = \"scores_\"+str(index)\n",
    "    savemat(os.path.join(score_path, sc_file), { 'genuine':scores_geniue, 'spoof':scores_spoof, 'eer' : a*100 })\n",
    "    savemat(os.path.join(prob_path, prob_file), { 'genuine':prob_genuine, 'spoof':prob_spoof })\n",
    "    \n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Training and testing accuracy\n",
    "session.run(init)\n",
    "\n",
    "if not os.path.exists(model_path):\n",
    "    os.makedirs(model_path)\n",
    "    \n",
    "if not os.path.exists(dev_sp_path):\n",
    "    os.makedirs(dev_sp_path)\n",
    "    \n",
    "batches = (len(list(os.listdir(features_path))))//2\n",
    "    \n",
    "for i in range(0,100):\n",
    "    for j in range(0,batches):  #number of batches\n",
    "        filename = \"batch_\"+str(j)+\"_features.npy\"\n",
    "        labelsname= \"batch_\"+str(j)+\"_labels.npy\"\n",
    "        \n",
    "        features = np.load(os.path.join(features_path, filename))\n",
    "        labels = np.load(os.path.join(features_path, labelsname))\n",
    "\n",
    "        train_step.run(session = session,feed_dict={ x:features, y:labels, keep_prob:0.5})\n",
    "        \n",
    "   \n",
    "    print(\"iteration   \",(i+1),\"\\n\")\n",
    "#     test_accuracy = accuracy.eval(session = session, feed_dict = { x:test_features, y:test_labels })\n",
    "#     print(\"iteration   \",(i+1),\"   dev_accuracy    \",test_accuracy)    \n",
    "#     print()\n",
    "    save_path = saver.save(session, model_path +str(i))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver(max_to_keep=None)\n",
    "\n",
    "with tf.Session() as session:\n",
    "    for i in range(0,100):\n",
    "        saver.restore(session, model_path +str(i))\n",
    "        print(\"iteration  \",(i+1))\n",
    "        eer_dev = get_EER(development_path, (i+1), dev_sp_path)\n",
    "        print(\" develop eer  \",eer_dev)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}