Image_Captioning_Flickr8K.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Image Captioning Flickr8K.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "f1LDve_jVz7x",
        "6did2gs4YgLZ",
        "KCF7dEuMemhG",
        "z1UnUVx6ewd4",
        "YGJcbdMrjQjU",
        "Lg47uhWUbUDW",
        "TX1eB38Vp-ZI",
        "xD-ldO6isVuL",
        "4unTKwvWsAMJ",
        "nf3aAJaGJU1u",
        "TJ58TTQWJCqD",
        "QAVJEWkhJFjH",
        "xgE1RWvzLb4u",
        "tPjpcFVaJK7q",
        "nQFV8f59NvzR"
      ],
      "toc_visible": true,
      "mount_file_id": "1ZOA195yLGo2OtVDMALI_pc5mq2lSeMVg",
      "authorship_tag": "ABX9TyOqwWbkyOEAW1F28cI1MTEr",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/radonys/Image-Captioning/blob/main/Image_Captioning_Flickr8K.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F6y6KmrgU6FD"
      },
      "source": [
        "# Image Captioning using Flickr8K Dataset\n",
        "\n",
        "### BUDT758 - Big Data and Artificial Intelligence\n",
        "\n",
        "##### Team Members: Aditya Kismatrao, Manas Mishra, Pratik Pandey & Yash Srivastava"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JRPPCl0GVXjF"
      },
      "source": [
        "Google Colaboratory Link: https://colab.research.google.com/drive/1ZOA195yLGo2OtVDMALI_pc5mq2lSeMVg?usp=sharing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f1LDve_jVz7x"
      },
      "source": [
        "### Import Modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lclA91PZUqiD"
      },
      "source": [
        "import os\n",
        "import requests\n",
        "import zipfile\n",
        "import io\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import datetime\n",
        "import matplotlib.pyplot as plt\n",
        "from pickle import dump, load\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "\n",
        "import tensorflow as tf\n",
        "from keras.applications.vgg16 import VGG16\n",
        "from keras.preprocessing import image\n",
        "from keras.models import Model, load_model\n",
        "from keras.applications.vgg16 import preprocess_input\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.utils import plot_model\n",
        "from keras.layers import Input\n",
        "from keras.layers import Dense\n",
        "from keras.layers import LSTM\n",
        "from keras.layers import Embedding\n",
        "from keras.layers import Dropout\n",
        "from keras.layers.merge import add\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from keras.callbacks import ModelCheckpoint"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-QnSpmRVFuHc"
      },
      "source": [
        "%load_ext tensorboard\r\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6did2gs4YgLZ"
      },
      "source": [
        "### Flickr8K Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7JfaDcLmV-FV"
      },
      "source": [
        "images = \"https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_Dataset.zip\"\n",
        "text = \"https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_text.zip\"\n",
        "\n",
        "if not os.path.exists(\"/content/FlickrImages/\"):\n",
        "\n",
        "  r = requests.get(images)\n",
        "  z = zipfile.ZipFile(io.BytesIO(r.content))\n",
        "  z.extractall(\"/content/FlickrImages/\")\n",
        "\n",
        "if not os.path.exists(\"/content/FlickrText/\"):\n",
        "\n",
        "  r = requests.get(text)\n",
        "  z = zipfile.ZipFile(io.BytesIO(r.content))\n",
        "  z.extractall(\"/content/FlickrText/\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KCF7dEuMemhG"
      },
      "source": [
        "### Data Cleaning and Processing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z1UnUVx6ewd4"
      },
      "source": [
        "#### Image-Caption Dictionary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3oYwRo-ceqPA"
      },
      "source": [
        "captions_path = \"/content/FlickrText/Flickr8k.token.txt\"\n",
        "\n",
        "image_captions = dict()\n",
        "\n",
        "with open(captions_path) as file:\n",
        "\n",
        "  captions = file.read()\n",
        "  \n",
        "  for caption in captions.split('\\n'):\n",
        "\n",
        "    if caption != '':\n",
        "      \n",
        "      splits = caption.split(\" \")\n",
        "\n",
        "      image_id = splits[0].split(\".\")[0]\n",
        "      string = ' '.join(splits[1:])\n",
        "\n",
        "      if image_id in image_captions:\n",
        "        image_captions[image_id].append(string)\n",
        "\n",
        "      else:\n",
        "        image_captions[image_id] = [string]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YGJcbdMrjQjU"
      },
      "source": [
        "#### Training and Validation Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTwrNgMMk0EG"
      },
      "source": [
        "def read_data(path):\n",
        "\n",
        "  data = list()\n",
        "\n",
        "  with open(path) as file:\n",
        "\n",
        "    images = file.read()\n",
        "\n",
        "    for image in images.split(\"\\n\"):\n",
        "      \n",
        "      if image != '':\n",
        "        data.append(image.split(\".\")[0])\n",
        "\n",
        "  return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R32RDMPfjSUE"
      },
      "source": [
        "train_path = \"/content/FlickrText/Flickr_8k.trainImages.txt\"\n",
        "val_path = \"/content/FlickrText/Flickr_8k.devImages.txt\"\n",
        "\n",
        "train_images = read_data(train_path)\n",
        "val_images = read_data(val_path)\n",
        "\n",
        "print(\"Number of Training Samples:\", len(train_images))\n",
        "print(\"Number of Validation Samples:\", len(val_images))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lg47uhWUbUDW"
      },
      "source": [
        "#### Clean Text\n",
        "\n",
        "- Clean Text by removing bad symbols and stopwords\n",
        "- Reference: https://github.com/radonys/Reddit-Flair-Detector/blob/master/Jupyter%20Notebooks/Reddit_Flair_Detector.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZYFXrY40bVYJ"
      },
      "source": [
        "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\n",
        "BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')\n",
        "STOPWORDS = set(stopwords.words('english'))\n",
        "\n",
        "def clean_text(text):\n",
        "   \n",
        "    text = BeautifulSoup(text, \"lxml\").text\n",
        "    text = text.lower()\n",
        "    text = REPLACE_BY_SPACE_RE.sub(' ', text)\n",
        "    text = BAD_SYMBOLS_RE.sub('', text)\n",
        "    text = ' '.join(word for word in text.split() if word.isalpha())\n",
        "\n",
        "    return text\n",
        "\n",
        "for key in image_captions:\n",
        "  image_captions[key] = [clean_text(caption) for caption in image_captions[key]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ehqpn48Ef3iQ"
      },
      "source": [
        "#### Text Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tf_SoGJif7am"
      },
      "source": [
        "def tokenize(captions):\n",
        "\n",
        "  caption_list = list()\n",
        "\n",
        "  for key in image_captions:\n",
        "    for caption in image_captions[key]:\n",
        "      caption_list.append(caption)\n",
        "\n",
        "  tokenizer = Tokenizer()\n",
        "  tokenizer.fit_on_texts(caption_list)\n",
        "\n",
        "  vocab_size = len(tokenizer.word_index) + 1\n",
        "  max_length = max(len(caption.split()) for caption in caption_list)\n",
        "\n",
        "  return tokenizer, vocab_size, max_length"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TX1eB38Vp-ZI"
      },
      "source": [
        "#### Start and End Identifiers for Captions\n",
        "\n",
        "Start Identifier: \"startcap\"\n",
        "\n",
        "End Identifier: \"endcap\""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2q-w1qJjpcEX"
      },
      "source": [
        "def startend(image_captions, keys):\n",
        "\n",
        "  marked_captions = dict()\n",
        "\n",
        "  for key in image_captions:\n",
        "\n",
        "    if key in keys:\n",
        "\n",
        "      for caption in image_captions[key]:\n",
        "\n",
        "        caption = 'startcap ' + caption + ' endcap'\n",
        "\n",
        "        if key in marked_captions:\n",
        "          marked_captions[key].append(caption)\n",
        "\n",
        "        else:\n",
        "          marked_captions[key] = [caption]\n",
        "\n",
        "  return marked_captions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xD-ldO6isVuL"
      },
      "source": [
        "### Convolutional Neural Network: VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XGrW8y7Gsahy"
      },
      "source": [
        "model_cnn = VGG16()\n",
        "model_cnn = Model(inputs=model_cnn.inputs, outputs=model_cnn.layers[-2].output)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4unTKwvWsAMJ"
      },
      "source": [
        "#### Save Train and Validation CNN Features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gbBk-gBWsd25"
      },
      "source": [
        "def preprocess(image_path):\n",
        "\n",
        "    img = image.load_img(image_path, target_size=(224, 224))\n",
        "    \n",
        "    img = image.img_to_array(img)\n",
        "\n",
        "    x = img.reshape((1, img.shape[0], img.shape[1], img.shape[2]))\n",
        "\n",
        "    x = preprocess_input(x)\n",
        "\n",
        "    return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6I9SG_pEsEJG"
      },
      "source": [
        "def encode(image):\n",
        "\n",
        "    image = preprocess(image)\n",
        "\n",
        "    features = model_cnn.predict(image)\n",
        "\n",
        "    return features"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2wjxFhpfudoH"
      },
      "source": [
        "images_path = \"/content/FlickrImages/Flicker8k_Dataset/\"\n",
        "\n",
        "train_images_encoded = dict()\n",
        "\n",
        "for img in train_images:\n",
        "  train_images_encoded[img] = encode(images_path + img + '.jpg')\n",
        "\n",
        "with open(\"/content/train_image_features.pkl\", \"wb\") as file:\n",
        "    dump(train_images_encoded, file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wehzvC169lWq"
      },
      "source": [
        "val_images_encoded = dict()\n",
        "\n",
        "for img in val_images:\n",
        "  val_images_encoded[img] = encode(images_path + img + '.jpg')\n",
        "\n",
        "with open(\"/content/val_image_features.pkl\", \"wb\") as file:\n",
        "    dump(val_images_encoded, file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nf3aAJaGJU1u"
      },
      "source": [
        "### Variable Declaration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xm_kTPx2JXoO"
      },
      "source": [
        "epochs = 10\n",
        "batch_size = 32"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TJ58TTQWJCqD"
      },
      "source": [
        "### Model Definition"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l_DOodLqJEu_"
      },
      "source": [
        "def image_captioning(vocab_size, max_length):\n",
        "\n",
        "  #Feature Extractor (FE)\n",
        "\tinput1 = Input(shape=(4096,))\n",
        "\tfe_drop = Dropout(0.5)(input1)\n",
        "\tfe_fc = Dense(256, activation='relu')(fe_drop)\n",
        " \n",
        "  #Sequential Model (Captions)\n",
        "\tinput2 = Input(shape=(max_length,))\n",
        "\tse_embed = Embedding(vocab_size, 256, mask_zero=True)(input2)\n",
        "\tse_drop = Dropout(0.5)(se_embed)\n",
        "\tse_lstm = LSTM(256)(se_drop)\n",
        " \n",
        "  #Combine Features\n",
        "\tcombine = add([fe_fc, se_lstm])\n",
        "\tcombine_fc = Dense(256, activation='relu')(combine)\n",
        "\toutput = Dense(vocab_size, activation='softmax')(combine_fc)\n",
        " \n",
        "\tmodel = Model(inputs=[input1, input2], outputs=output)\n",
        "\t\n",
        "\treturn model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QAVJEWkhJFjH"
      },
      "source": [
        "### Data Generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liazPSBuJID2"
      },
      "source": [
        "def data_generator(captions, images, tokenizer, vocab_size, max_length, batch_size):\n",
        "\t\n",
        "  X1, X2, y = list(), list(), list()\n",
        "  counter = 0\n",
        "  \n",
        "  while 1:\n",
        "\n",
        "    for key in captions:\n",
        "      \n",
        "      counter += 1\n",
        "\n",
        "      for caption in captions[key]:\n",
        "\n",
        "        caption_sequence = tokenizer.texts_to_sequences([caption])[0]\n",
        "        \n",
        "        for i in range(1, len(caption_sequence)):\n",
        "          \n",
        "          input_seq, output_seq = caption_sequence[:i], caption_sequence[i]\n",
        "        \n",
        "          input_seq = pad_sequences([input_seq], maxlen=max_length)[0]\n",
        "          output_seq = to_categorical([output_seq], num_classes=vocab_size)[0]\n",
        "\n",
        "          X1.append(images[key][0])\n",
        "          X2.append(input_seq)\n",
        "          y.append(output_seq)\n",
        "\n",
        "      if counter == batch_size:\n",
        "\n",
        "        yield ([np.array(X1), np.array(X2)], np.array(y))\n",
        "\n",
        "        X1, X2, y = list(), list(), list()\n",
        "        counter = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xgE1RWvzLb4u"
      },
      "source": [
        "### Model Compilation and Train/Validation Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KTjbcrVZbKJ4"
      },
      "source": [
        "#train_image_features = load(open(\"/content/train_image_features.pkl\", \"rb\"))\n",
        "train_captions = startend(image_captions, train_images)\n",
        "\n",
        "tokenizer, vocab_size, max_length = tokenize(train_captions)\n",
        " \n",
        "#val_image_features = load(open(\"/content/val_image_features.pkl\", \"rb\"))\n",
        "val_captions = startend(image_captions, val_images)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ehVf3sHLfxs"
      },
      "source": [
        "model = image_captioning(vocab_size, max_length)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tPjpcFVaJK7q"
      },
      "source": [
        "### Model Training and Validation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FgmBpXWNJQG_"
      },
      "source": [
        "train_steps = len(train_captions)//batch_size\n",
        "val_steps = len(val_captions)//batch_size\n",
        "\n",
        "logdir = os.path.join(\"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
        "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)\n",
        "\n",
        "for epoch in range(0, epochs):\n",
        "\n",
        "  print(\"Epoch:\", epoch)\n",
        "\n",
        "  train_generator = data_generator(train_captions, train_images_encoded, tokenizer, vocab_size, max_length, batch_size)\n",
        "  val_generator = data_generator(val_captions, val_images_encoded, tokenizer, vocab_size, max_length, batch_size)\n",
        "\n",
        "  model.fit(train_generator, epochs=1, verbose=1, steps_per_epoch=train_steps, validation_data=val_generator, validation_steps=val_steps, callbacks=[tensorboard_callback])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YN5e40-lRDIw"
      },
      "source": [
        "model.save(\"model_vgg.h5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fN3a98kRNzK"
      },
      "source": [
        "%tensorboard --logdir logs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nQFV8f59NvzR"
      },
      "source": [
        "### Model Output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17AcqfpIZW7L"
      },
      "source": [
        "def int_to_word(integer, tokenizer):\n",
        "  \n",
        "  for word, index in tokenizer.word_index.items():\n",
        "   \n",
        "   if index == integer:\n",
        "     return word\n",
        "\t\n",
        "  return None"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_o5lP4G7N_zu"
      },
      "source": [
        "def model_output(image_path, model, tokenizer, max_length):\n",
        "  \n",
        "  caption = 'startcap'\n",
        "  image_feature = encode(image_path)\n",
        "  \n",
        "  for i in range(0, max_length):\n",
        "   \n",
        "    sequence = tokenizer.texts_to_sequences([caption])[0]\n",
        "\n",
        "    sequence = pad_sequences([sequence], maxlen=max_length)\n",
        "\n",
        "    y = model.predict([image_feature, sequence], verbose=0)\n",
        "    y = np.argmax(y)\n",
        "\n",
        "    word = int_to_word(y, tokenizer)\n",
        "\n",
        "    if word is None:\n",
        "      break\n",
        "\n",
        "    caption += ' ' + word\n",
        "\n",
        "    if word == 'endcap':\n",
        "      break\n",
        "\n",
        "  return caption"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WNs53zg_wSV"
      },
      "source": [
        "model = load_model('/content/model_vgg.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RkwJqoEAaTws"
      },
      "source": [
        "image_path = \"/content/FlickrImages/Flicker8k_Dataset/1000268201_693b08cb0e.jpg\"\n",
        "caption = model_output(image_path, model, tokenizer, max_length)\n",
        "\n",
        "caption = caption.split(\" \")[1:-1]\n",
        "caption = \" \".join(caption)\n",
        "\n",
        "x = plt.imread(image_path)\n",
        "plt.imshow(x)\n",
        "print(\"Predicted Caption:\", caption)\n",
        "print(\"Actual Caption(s):\", image_captions[image_path.split(\"/\")[-1].split(\".\")[0]])"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}