Recurrence relations, p=2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Fibonacci sequence (p=2).ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "brTA-k9MBeVU",
        "colab_type": "code",
        "outputId": "ea9dd86f-1dfb-4c68-f5ef-67d15f060609",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Импортируем библиотеки\n",
        "\n",
        "%tensorflow_version 1.x\n",
        "import numpy as np \n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import LSTM\n",
        "from tensorflow.keras.layers import Dense\n",
        "from sklearn.metrics import mean_squared_error"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsRnuPkDBhnb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_sequence(sequence, n_steps):\n",
        "    n = len(sequence)\n",
        "    X, y = list(), list()\n",
        "    for i in range(n):\n",
        "        end_ix = i + n_steps\n",
        "        if end_ix > n-1:\n",
        "            break\n",
        "        seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]\n",
        "        X.append(seq_x)\n",
        "        y.append(seq_y)\n",
        "    return np.array(X), np.array(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z_d7XPCl9uYL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_difference(true_sequence, predict_values, unit):\n",
        "    print(f'Prediction of model with {unit} units:')\n",
        "    df = pd.DataFrame({'real value': true_sequence, 'predicted value': np.round(predict_values, 3), 'difference': np.round(abs(true_sequence-predict_values), 1)})\n",
        "    with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
        "        print(df)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mAsAPMA8gaHY",
        "colab_type": "text"
      },
      "source": [
        "# **Data generation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvV05Wvu-D-x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fibonacci(start=(0, 1), num=10):\n",
        "    x = [start[0], start[1]]\n",
        "    for i in range(2, num+1):\n",
        "        x.append(x[i-1] + x[i-2])\n",
        "    return np.array(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T_HCtSs84KqJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_data(n_steps=4):\n",
        "    \n",
        "    num = 25\n",
        "    batch_size = 2000\n",
        "    i = 0\n",
        "    \n",
        "    X = np.empty((batch_size, n_steps))\n",
        "    y = np.empty((batch_size, 1))\n",
        "\n",
        "    start_val = np.array([0, 1])\n",
        "\n",
        "    while i < batch_size:\n",
        "    \n",
        "        array = fibonacci(start=(start_val[0], start_val[1]), num=num)\n",
        "        j = 0\n",
        "\n",
        "        while j <= num - (n_steps + 1):\n",
        "            if i >= batch_size: break\n",
        "            X[i, :] = array[j:n_steps+j]\n",
        "            y[i, :] = array[j+n_steps] \n",
        "            i += 1\n",
        "            j += 1\n",
        "        \n",
        "        start_val += 1\n",
        "\n",
        "    _, index = np.unique(X, axis=0, return_index=True)\n",
        "    X, y = X[index], y[index]\n",
        "    X = X.reshape((X.shape[0], X.shape[1], 1))\n",
        "    return X, y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZxsSYkITkmLj",
        "colab_type": "code",
        "outputId": "2d6a806e-bc8c-4d06-94d4-f88a66a6ca2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "n_steps = 4\n",
        "X_train, y_train = generate_data(n_steps)\n",
        "print(f'X_train shape is {X_train.shape}')\n",
        "print(f'y_train shape is {y_train.shape}')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X_train shape is (1961, 4, 1)\n",
            "y_train shape is (1961, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gAipwMcQLHoq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "94e5f9f0-0c7b-4706-b9c8-687230a1bd2e"
      },
      "source": [
        "np.min(X_train), np.max(y_train)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.0, 7098718.0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W18N19mph1Rx",
        "colab_type": "text"
      },
      "source": [
        "## **Model with one LSTM layer**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bLtcQELn-UZN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(true_sequence, model):\n",
        "    predict_values = true_sequence[:n_steps]\n",
        "    k = n_steps\n",
        "    length = true_sequence.size\n",
        "    while k != length:\n",
        "        X = predict_values[-n_steps::]\n",
        "        X = X.reshape((1, n_steps, 1))\n",
        "        f_x = model.predict(X, verbose=0) \n",
        "        predict_values = np.append(predict_values, f_x)\n",
        "        k += 1\n",
        "    return predict_values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Et5WoaFM-VYX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequence = fibonacci(start=(2, 2), num=25)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g9r8hz6O9qXR",
        "colab_type": "code",
        "outputId": "0a8b87af-bb5b-41a3-9134-8622acc27f49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "units_number = list(range(1, 11)) + list(range(15, 65, 5))\n",
        "losses = []\n",
        "min_loss = None\n",
        "min_unit = None\n",
        "\n",
        "for unit in units_number: \n",
        "    model = Sequential()\n",
        "    model.add(LSTM(unit, activation='softplus', input_shape=(n_steps, 1)))\n",
        "    model.add(Dense(1))\n",
        "    model.compile(optimizer='adam', loss='mse')\n",
        "\n",
        "    model.fit(X_train, y_train, epochs=800, validation_split=0.2, verbose=0)\n",
        "    \n",
        "    predict_y = predict(sequence, model)\n",
        "    mse = mean_squared_error(sequence, predict_y)\n",
        "\n",
        "    if min_loss is None or mse < min_loss:\n",
        "         min_unit = unit\n",
        "         min_loss = mse\n",
        "         best_model = model\n",
        "    losses.append(mse)\n",
        "    \n",
        "    print(f'Units in LSTM layer: {unit}, MSE is: {np.round(mse, 4)}')\n",
        "    if unit in (5, 6, 7, 8, 9, 40, 50):\n",
        "        plot_difference(sequence, predict_y, unit)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Units in LSTM layer: 1, MSE is: 748175.0701\n",
            "Units in LSTM layer: 2, MSE is: 356702.2989\n",
            "Units in LSTM layer: 3, MSE is: 984858.6574\n",
            "Units in LSTM layer: 4, MSE is: 262642221.6964\n",
            "Units in LSTM layer: 5, MSE is: 492245.5523\n",
            "Prediction of model with 5 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.089         0.1\n",
            "5           16           15.456         0.5\n",
            "6           26           25.481         0.5\n",
            "7           42           41.611         0.4\n",
            "8           68           67.149         0.9\n",
            "9          110          108.057         1.9\n",
            "10         178          176.572         1.4\n",
            "11         288          283.999         4.0\n",
            "12         466          458.563         7.4\n",
            "13         754          746.112         7.9\n",
            "14        1220         1206.064        13.9\n",
            "15        1974         1948.173        25.8\n",
            "16        3194         3153.545        40.5\n",
            "17        5168         5108.829        59.2\n",
            "18        8362         8261.968       100.0\n",
            "19       13530        13364.426       165.6\n",
            "20       21892        21631.436       260.6\n",
            "21       35422        35009.094       412.9\n",
            "22       57314        56637.973       676.0\n",
            "23       92736        91645.164      1090.8\n",
            "24      150050       148308.750      1741.2\n",
            "25      242786       239986.438      2799.6\n",
            "Units in LSTM layer: 6, MSE is: 2010049.3323\n",
            "Prediction of model with 6 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.323         0.3\n",
            "5           16           16.301         0.3\n",
            "6           26           26.303         0.3\n",
            "7           42           42.888         0.9\n",
            "8           68           69.218         1.2\n",
            "9          110          111.386         1.4\n",
            "10         178          180.898         2.9\n",
            "11         288          296.094         8.1\n",
            "12         466          475.944         9.9\n",
            "13         754          770.682        16.7\n",
            "14        1220         1247.880        27.9\n",
            "15        1974         2021.014        47.0\n",
            "16        3194         3267.557        73.6\n",
            "17        5168         5288.263       120.3\n",
            "18        8362         8557.296       195.3\n",
            "19       13530        13846.981       317.0\n",
            "20       21892        22403.166       511.2\n",
            "21       35422        36250.645       828.6\n",
            "22       57314        58655.254      1341.3\n",
            "23       92736        94906.836      2170.8\n",
            "24      150050       153561.703      3511.7\n",
            "25      242786       248470.000      5684.0\n",
            "Units in LSTM layer: 7, MSE is: 488578.378\n",
            "Prediction of model with 7 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.140         0.1\n",
            "5           16           15.803         0.2\n",
            "6           26           26.142         0.1\n",
            "7           42           41.658         0.3\n",
            "8           68           67.457         0.5\n",
            "9          110          109.050         1.0\n",
            "10         178          176.143         1.9\n",
            "11         288          284.858         3.1\n",
            "12         466          460.476         5.5\n",
            "13         754          745.638         8.4\n",
            "14        1220         1206.057        13.9\n",
            "15        1974         1950.485        23.5\n",
            "16        3194         3157.249        36.8\n",
            "17        5168         5108.345        59.7\n",
            "18        8362         8265.362        96.6\n",
            "19       13530        13373.621       156.4\n",
            "20       21892        21639.449       252.6\n",
            "21       35422        35013.148       408.9\n",
            "22       57314        56652.543       661.5\n",
            "23       92736        91665.773      1070.2\n",
            "24      150050       148318.406      1731.6\n",
            "25      242786       239984.000      2802.0\n",
            "Units in LSTM layer: 8, MSE is: 1230933.9377\n",
            "Prediction of model with 8 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.600         0.6\n",
            "5           16           16.041         0.0\n",
            "6           26           26.590         0.6\n",
            "7           42           43.193         1.2\n",
            "8           68           69.426         1.4\n",
            "9          110          111.632         1.6\n",
            "10         178          181.251         3.3\n",
            "11         288          293.455         5.5\n",
            "12         466          475.086         9.1\n",
            "13         754          767.540        13.5\n",
            "14        1220         1241.312        21.3\n",
            "15        1974         2010.803        36.8\n",
            "16        3194         3253.026        59.0\n",
            "17        5168         5261.970        94.0\n",
            "18        8362         8514.367       152.4\n",
            "19       13530        13778.996       249.0\n",
            "20       21892        22293.352       401.4\n",
            "21       35422        36069.887       647.9\n",
            "22       57314        58363.727      1049.7\n",
            "23       92736        94436.578      1700.6\n",
            "24      150050       152798.875      2748.9\n",
            "25      242786       247232.750      4446.8\n",
            "Units in LSTM layer: 9, MSE is: 132227.2251\n",
            "Prediction of model with 9 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10            9.962         0.0\n",
            "5           16           15.662         0.3\n",
            "6           26           25.827         0.2\n",
            "7           42           41.508         0.5\n",
            "8           68           67.599         0.4\n",
            "9          110          108.867         1.1\n",
            "10         178          176.371         1.6\n",
            "11         288          286.134         1.9\n",
            "12         466          462.802         3.2\n",
            "13         754          750.423         3.6\n",
            "14        1220         1211.455         8.5\n",
            "15        1974         1961.826        12.2\n",
            "16        3194         3174.655        19.3\n",
            "17        5168         5137.842        30.2\n",
            "18        8362         8309.979        52.0\n",
            "19       13530        13448.207        81.8\n",
            "20       21892        21760.523       131.5\n",
            "21       35422        35209.715       212.3\n",
            "22       57314        56967.062       346.9\n",
            "23       92736        92178.445       557.6\n",
            "24      150050       149149.547       900.5\n",
            "25      242786       241329.094      1456.9\n",
            "Units in LSTM layer: 10, MSE is: 7209025.0679\n",
            "Units in LSTM layer: 15, MSE is: 369294.9763\n",
            "Units in LSTM layer: 20, MSE is: 67.4472\n",
            "Units in LSTM layer: 25, MSE is: 67953.1078\n",
            "Units in LSTM layer: 30, MSE is: 94989.186\n",
            "Units in LSTM layer: 35, MSE is: 61824.6808\n",
            "Units in LSTM layer: 40, MSE is: 9271.3965\n",
            "Prediction of model with 40 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.026         0.0\n",
            "5           16           15.915         0.1\n",
            "6           26           26.047         0.0\n",
            "7           42           42.021         0.0\n",
            "8           68           68.029         0.0\n",
            "9          110          109.970         0.0\n",
            "10         178          178.162         0.2\n",
            "11         288          288.203         0.2\n",
            "12         466          465.233         0.8\n",
            "13         754          754.941         0.9\n",
            "14        1220         1220.739         0.7\n",
            "15        1974         1977.417         3.4\n",
            "16        3194         3195.560         1.6\n",
            "17        5168         5184.548        16.5\n",
            "18        8362         8373.014        11.0\n",
            "19       13530        13550.730        20.7\n",
            "20       21892        21927.756        35.8\n",
            "21       35422        35481.477        59.5\n",
            "22       57314        57402.148        88.1\n",
            "23       92736        92882.977       147.0\n",
            "24      150050       150289.266       239.3\n",
            "25      242786       243171.719       385.7\n",
            "Units in LSTM layer: 45, MSE is: 6556.5274\n",
            "Units in LSTM layer: 50, MSE is: 18390611.9374\n",
            "Prediction of model with 50 units:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.297         0.3\n",
            "5           16           16.695         0.7\n",
            "6           26           27.398         1.4\n",
            "7           42           44.492         2.5\n",
            "8           68           72.418         4.4\n",
            "9          110          117.507         7.5\n",
            "10         178          190.265        12.3\n",
            "11         288          308.899        20.9\n",
            "12         466          498.672        32.7\n",
            "13         754          807.138        53.1\n",
            "14        1220         1306.289        86.3\n",
            "15        1974         2114.029       140.0\n",
            "16        3194         3419.794       225.8\n",
            "17        5168         5533.777       365.8\n",
            "18        8362         8954.126       592.1\n",
            "19       13530        14488.062       958.1\n",
            "20       21892        23441.787      1549.8\n",
            "21       35422        37930.074      2508.1\n",
            "22       57314        61372.254      4058.3\n",
            "23       92736        99302.219      6566.2\n",
            "24      150050       160674.234     10624.2\n",
            "25      242786       259976.688     17190.7\n",
            "Units in LSTM layer: 55, MSE is: 4971548.86\n",
            "Units in LSTM layer: 60, MSE is: 85468.4609\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oPqAPiuoiM1e",
        "colab_type": "text"
      },
      "source": [
        "# **Testing best model performance**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HHjPa5smN4Bj",
        "colab_type": "text"
      },
      "source": [
        "$F_n = F_{n-1} + F_{n-2}, n >= 2$  \n",
        "Пусть $F_0 = 0, F_1 = 1$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FR3oB8ZhYdlt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "outputId": "2b30f84b-bbb7-457e-f27e-7b8871f5e54a"
      },
      "source": [
        "sequence = fibonacci(start=(0, 1), num=25)\n",
        "predict_y = predict(sequence, best_model)\n",
        "mse = mean_squared_error(sequence, predict_y)\n",
        "print(f'mse is {mse}')\n",
        "plot_difference(sequence, predict_y, min_unit)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mse is 557.3639900387494\n",
            "Prediction of model with 20 units:\n",
            "    real value  predicted value  difference\n",
            "0            0            0.000         0.0\n",
            "1            1            1.000         0.0\n",
            "2            1            1.000         0.0\n",
            "3            2            2.000         0.0\n",
            "4            3            3.002         0.0\n",
            "5            5            4.970         0.0\n",
            "6            8            8.019         0.0\n",
            "7           13           13.038         0.0\n",
            "8           21           21.046         0.0\n",
            "9           34           34.058         0.1\n",
            "10          55           55.123         0.1\n",
            "11          89           89.145         0.1\n",
            "12         144          144.192         0.2\n",
            "13         233          233.350         0.4\n",
            "14         377          377.395         0.4\n",
            "15         610          610.937         0.9\n",
            "16         987          988.015         1.0\n",
            "17        1597         1598.949         1.9\n",
            "18        2584         2587.483         3.5\n",
            "19        4181         4186.324         5.3\n",
            "20        6765         6773.510         8.5\n",
            "21       10946        10959.989        14.0\n",
            "22       17711        17733.439        22.4\n",
            "23       28657        28693.152        36.2\n",
            "24       46368        46426.555        58.6\n",
            "25       75025        75119.539        94.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yxp-0OnzOvGc",
        "colab_type": "text"
      },
      "source": [
        "Пусть $F_0 = 4, F_1=5$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fyLe1vXdYgyX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "outputId": "45db5442-cd02-4d5d-89b8-deda4a179939"
      },
      "source": [
        "sequence = fibonacci(start=(4, 5), num=25)\n",
        "predict_y = predict(sequence, best_model)\n",
        "mse = mean_squared_error(sequence, predict_y)\n",
        "print(f'mse is {mse}')\n",
        "plot_difference(sequence, predict_y, min_unit)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mse is 11191.050378523614\n",
            "Prediction of model with 20 units:\n",
            "    real value  predicted value  difference\n",
            "0            4            4.000         0.0\n",
            "1            5            5.000         0.0\n",
            "2            9            9.000         0.0\n",
            "3           14           14.000         0.0\n",
            "4           23           23.038         0.0\n",
            "5           37           36.998         0.0\n",
            "6           60           59.985         0.0\n",
            "7           97           96.986         0.0\n",
            "8          157          156.956         0.0\n",
            "9          254          253.786         0.2\n",
            "10         411          410.652         0.3\n",
            "11         665          664.642         0.4\n",
            "12        1076         1074.960         1.0\n",
            "13        1741         1739.680         1.3\n",
            "14        2817         2815.103         1.9\n",
            "15        4558         4554.625         3.4\n",
            "16        7375         7369.477         5.5\n",
            "17       11933        11924.250         8.8\n",
            "18       19308        19293.635        14.4\n",
            "19       31241        31217.623        23.4\n",
            "20       50549        50511.215        37.8\n",
            "21       81790        81728.648        61.4\n",
            "22      132339       132239.406        99.6\n",
            "23      214129       213967.578       161.4\n",
            "24      346468       346206.156       261.8\n",
            "25      560597       560172.312       424.7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ubIAoa8WYpU6",
        "colab_type": "text"
      },
      "source": [
        "$F_0=4, F_1=7$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qxvg4dU9Yklb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "outputId": "86a89ae8-d4cb-4076-ca6d-06c7731b855c"
      },
      "source": [
        "sequence = fibonacci(start=(4, 7), num=25)\n",
        "predict_y = predict(sequence, best_model)\n",
        "mse = mean_squared_error(sequence, predict_y)\n",
        "print(f'mse is {mse}')\n",
        "plot_difference(sequence, predict_y, min_unit)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mse is 18052.633686379126\n",
            "Prediction of model with 20 units:\n",
            "    real value  predicted value  difference\n",
            "0            4            4.000         0.0\n",
            "1            7            7.000         0.0\n",
            "2           11           11.000         0.0\n",
            "3           18           18.000         0.0\n",
            "4           29           29.019         0.0\n",
            "5           47           46.987         0.0\n",
            "6           76           75.967         0.0\n",
            "7          123          122.956         0.0\n",
            "8          199          198.807         0.2\n",
            "9          322          321.718         0.3\n",
            "10         521          520.692         0.3\n",
            "11         843          842.286         0.7\n",
            "12        1364         1362.781         1.2\n",
            "13        2207         2205.563         1.4\n",
            "14        3571         3568.419         2.6\n",
            "15        5778         5773.630         4.4\n",
            "16        9349         9342.147         6.9\n",
            "17       15127        15115.804        11.2\n",
            "18       24476        24457.676        18.3\n",
            "19       39603        39573.398        29.6\n",
            "20       64079        64030.980        48.0\n",
            "21      103682       103604.000        78.0\n",
            "22      167761       167634.547       126.5\n",
            "23      271443       271237.938       205.1\n",
            "24      439204       438871.438       332.6\n",
            "25      710647       710107.625       539.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3qPl_V3hfOY",
        "colab_type": "text"
      },
      "source": [
        "# **Model with two LSTM layers**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WYA8uGcHhjsJ",
        "colab_type": "code",
        "outputId": "61fdc974-b8cc-437b-9878-9a6eea2b5c7a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "n_steps = 4\n",
        "X_train, y_train = generate_data(n_steps)\n",
        "print(f'X_train shape is {X_train.shape}')\n",
        "print(f'y_train shape is {y_train.shape}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X_train shape is (1961, 4, 1)\n",
            "y_train shape is (1961, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "krAIYPrYhu67",
        "colab_type": "code",
        "outputId": "738679ce-f61b-48f4-e3c5-469ae7e5f60c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "units_number = list(range(1, 11)) + list(range(15, 65, 5))\n",
        "losses = []\n",
        "min_loss = None\n",
        "min_unit = None\n",
        "\n",
        "for unit in units_number: \n",
        "    model = Sequential()\n",
        "    model.add(LSTM(unit, activation='softplus', return_sequences=True, input_shape=(n_steps, 1)))\n",
        "    model.add(LSTM(unit, activation='softplus'))\n",
        "    model.add(Dense(1))\n",
        "    model.compile(optimizer='adam', loss='mse')\n",
        "\n",
        "    model.fit(X_train, y_train, epochs=800, validation_split=0.2, verbose=0)\n",
        "    \n",
        "    predict_y = predict(sequence, model)\n",
        "    mse = mean_squared_error(sequence, predict_y)\n",
        "\n",
        "    if min_loss is None or mse < min_loss:\n",
        "         min_unit = unit\n",
        "         min_loss = mse\n",
        "         best_model = model\n",
        "    losses.append(mse)\n",
        "    \n",
        "    print(f'Units in LSTM layer: {unit}, MSE is:: {np.round(mse, 4)}')\n",
        "    if unit in (5, 6, 7, 8, 9, 40, 50):\n",
        "        plot_difference(sequence, predict_y, unit)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "Units in LSTM layer: 1, MSE is:: 543384195.084\n",
            "Units in LSTM layer: 2, MSE is:: 3666441733.9008\n",
            "Units in LSTM layer: 3, MSE is:: 20419857.0409\n",
            "Units in LSTM layer: 4, MSE is:: 2752561.5383\n",
            "Units in LSTM layer: 5, MSE is:: 35766405974.386\n",
            "Prediction of model with 5:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.980         1.0\n",
            "5           16           16.469         0.5\n",
            "6           26           26.989         1.0\n",
            "7           42           44.543         2.5\n",
            "8           68           72.580         4.6\n",
            "9          110          117.571         7.6\n",
            "10         178          191.963        14.0\n",
            "11         288          321.694        33.7\n",
            "12         466          554.667        88.7\n",
            "13         754         1060.980       307.0\n",
            "14        1220         1938.724       718.7\n",
            "15        1974         2993.669      1019.7\n",
            "16        3194         5304.425      2110.4\n",
            "17        5168        10850.222      5682.2\n",
            "18        8362        18822.646     10460.6\n",
            "19       13530        28500.740     14970.7\n",
            "20       21892        52246.039     30354.0\n",
            "21       35422       106941.820     71519.8\n",
            "22       57314       182027.281    124713.3\n",
            "23       92736       277411.000    184675.0\n",
            "24      150050       587363.062    437313.1\n",
            "25      242786      1068992.500    826206.5\n",
            "Units in LSTM layer: 6, MSE is:: 750497.8369\n",
            "Prediction of model with 6:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.357         0.4\n",
            "5           16           15.514         0.5\n",
            "6           26           26.515         0.5\n",
            "7           42           42.391         0.4\n",
            "8           68           68.474         0.5\n",
            "9          110          111.124         1.1\n",
            "10         178          180.490         2.5\n",
            "11         288          288.940         0.9\n",
            "12         466          476.546        10.5\n",
            "13         754          765.360        11.4\n",
            "14        1220         1235.232        15.2\n",
            "15        1974         1998.600        24.6\n",
            "16        3194         3245.026        51.0\n",
            "17        5168         5241.921        73.9\n",
            "18        8362         8475.981       114.0\n",
            "19       13530        13721.085       191.1\n",
            "20       21892        22218.074       326.1\n",
            "21       35422        35924.254       502.3\n",
            "22       57314        58123.305       809.3\n",
            "23       92736        94068.938      1332.9\n",
            "24      150050       152222.547      2172.5\n",
            "25      242786       246242.016      3456.0\n",
            "Units in LSTM layer: 7, MSE is:: 357701.7693\n",
            "Prediction of model with 7:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.559         0.6\n",
            "5           16           15.942         0.1\n",
            "6           26           26.203         0.2\n",
            "7           42           42.888         0.9\n",
            "8           68           69.065         1.1\n",
            "9          110          112.353         2.4\n",
            "10         178          180.169         2.2\n",
            "11         288          288.831         0.8\n",
            "12         466          471.207         5.2\n",
            "13         754          761.257         7.3\n",
            "14        1220         1231.913        11.9\n",
            "15        1974         1993.062        19.1\n",
            "16        3194         3225.754        31.8\n",
            "17        5168         5218.986        51.0\n",
            "18        8362         8444.773        82.8\n",
            "19       13530        13663.896       133.9\n",
            "20       21892        22108.793       216.8\n",
            "21       35422        35772.516       350.5\n",
            "22       57314        57880.930       566.9\n",
            "23       92736        93652.680       916.7\n",
            "24      150050       151532.203      1482.2\n",
            "25      242786       245182.406      2396.4\n",
            "Units in LSTM layer: 8, MSE is:: 822337.5155\n",
            "Prediction of model with 8:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.238         0.2\n",
            "5           16           16.370         0.4\n",
            "6           26           26.284         0.3\n",
            "7           42           42.665         0.7\n",
            "8           68           69.258         1.3\n",
            "9          110          111.897         1.9\n",
            "10         178          180.327         2.3\n",
            "11         288          292.239         4.2\n",
            "12         466          473.187         7.2\n",
            "13         754          764.998        11.0\n",
            "14        1220         1239.221        19.2\n",
            "15        1974         2003.526        29.5\n",
            "16        3194         3241.015        47.0\n",
            "17        5168         5246.884        78.9\n",
            "18        8362         8487.070       125.1\n",
            "19       13530        13731.067       201.1\n",
            "20       21892        22222.385       330.4\n",
            "21       35422        35952.262       530.3\n",
            "22       57314        58169.684       855.7\n",
            "23       92736        94129.602      1393.6\n",
            "24      150050       152297.703      2247.7\n",
            "25      242786       246418.906      3632.9\n",
            "Units in LSTM layer: 9, MSE is:: 19653.1426\n",
            "Prediction of model with 9:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10            9.842         0.2\n",
            "5           16           16.084         0.1\n",
            "6           26           26.124         0.1\n",
            "7           42           42.253         0.3\n",
            "8           68           68.146         0.1\n",
            "9          110          110.794         0.8\n",
            "10         178          178.544         0.5\n",
            "11         288          288.213         0.2\n",
            "12         466          467.140         1.1\n",
            "13         754          755.884         1.9\n",
            "14        1220         1222.874         2.9\n",
            "15        1974         1977.949         3.9\n",
            "16        3194         3201.726         7.7\n",
            "17        5168         5180.267        12.3\n",
            "18        8362         8381.227        19.2\n",
            "19       13530        13560.620        30.6\n",
            "20       21892        21943.561        51.6\n",
            "21       35422        35504.430        82.4\n",
            "22       57314        57446.160       132.2\n",
            "23       92736        92949.961       214.0\n",
            "24      150050       150398.703       348.7\n",
            "25      242786       243347.375       561.4\n",
            "Units in LSTM layer: 10, MSE is:: 209653.564\n",
            "Units in LSTM layer: 15, MSE is:: 794678.2065\n",
            "Units in LSTM layer: 20, MSE is:: 99203.4913\n",
            "Units in LSTM layer: 25, MSE is:: 79491.6395\n",
            "Units in LSTM layer: 30, MSE is:: 471420.9055\n",
            "Units in LSTM layer: 35, MSE is:: 150297.8982\n",
            "Units in LSTM layer: 40, MSE is:: 125029.7787\n",
            "Prediction of model with 40:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.173         0.2\n",
            "5           16           15.972         0.0\n",
            "6           26           26.025         0.0\n",
            "7           42           42.318         0.3\n",
            "8           68           67.937         0.1\n",
            "9          110          110.241         0.2\n",
            "10         178          179.126         1.1\n",
            "11         288          289.283         1.3\n",
            "12         466          468.140         2.1\n",
            "13         754          758.903         4.9\n",
            "14        1220         1228.221         8.2\n",
            "15        1974         1984.033        10.0\n",
            "16        3194         3211.649        17.6\n",
            "17        5168         5199.712        31.7\n",
            "18        8362         8411.718        49.7\n",
            "19       13530        13605.059        75.1\n",
            "20       21892        22019.350       127.3\n",
            "21       35422        35632.406       210.4\n",
            "22       57314        57646.980       333.0\n",
            "23       92736        93268.258       532.3\n",
            "24      150050       150927.016       877.0\n",
            "25      242786       244206.328      1420.3\n",
            "Units in LSTM layer: 45, MSE is:: 645099.4614\n",
            "Units in LSTM layer: 50, MSE is:: 1273.5504\n",
            "Prediction of model with 50:\n",
            "    real value  predicted value  difference\n",
            "0            2            2.000         0.0\n",
            "1            2            2.000         0.0\n",
            "2            4            4.000         0.0\n",
            "3            6            6.000         0.0\n",
            "4           10           10.034         0.0\n",
            "5           16           15.912         0.1\n",
            "6           26           26.031         0.0\n",
            "7           42           42.044         0.0\n",
            "8           68           68.013         0.0\n",
            "9          110          109.872         0.1\n",
            "10         178          177.893         0.1\n",
            "11         288          287.946         0.1\n",
            "12         466          465.661         0.3\n",
            "13         754          753.574         0.4\n",
            "14        1220         1219.300         0.7\n",
            "15        1974         1972.806         1.2\n",
            "16        3194         3192.012         2.0\n",
            "17        5168         5164.582         3.4\n",
            "18        8362         8356.456         5.5\n",
            "19       13530        13521.135         8.9\n",
            "20       21892        21877.787        14.2\n",
            "21       35422        35399.371        22.6\n",
            "22       57314        57278.215        35.8\n",
            "23       92736        92679.414        56.6\n",
            "24      150050       149960.656        89.3\n",
            "25      242786       242645.266       140.7\n",
            "Units in LSTM layer: 55, MSE is:: 943749.7862\n",
            "Units in LSTM layer: 60, MSE is:: 680261.0638\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NoZCRkoK5uqv",
        "colab_type": "text"
      },
      "source": [
        "# **Testing best model performance**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XAFdTNmJiBfi",
        "colab_type": "code",
        "outputId": "6b860870-616a-48d0-d30b-e07b8168e5c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        }
      },
      "source": [
        "sequence = fibonacci(start=(0, 1), num=25)\n",
        "predict_y = predict(sequence, best_model)\n",
        "mse = mean_squared_error(sequence, predict_y)\n",
        "print(f'mse is {mse}')\n",
        "plot_difference(sequence, predict_y, min_unit)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mse is 74231.00849568703\n",
            "Prediction of model with 50:\n",
            "    real value  predicted value  difference\n",
            "0            0            0.000         0.0\n",
            "1            1            1.000         0.0\n",
            "2            1            1.000         0.0\n",
            "3            2            2.000         0.0\n",
            "4            3            2.945         0.1\n",
            "5            5            4.928         0.1\n",
            "6            8            7.876         0.1\n",
            "7           13           12.806         0.2\n",
            "8           21           20.693         0.3\n",
            "9           34           33.504         0.5\n",
            "10          55           54.237         0.8\n",
            "11          89           87.791         1.2\n",
            "12         144          141.876         2.1\n",
            "13         233          229.632         3.4\n",
            "14         377          371.694         5.3\n",
            "15         610          601.046         9.0\n",
            "16         987          972.434        14.6\n",
            "17        1597         1574.029        23.0\n",
            "18        2584         2546.520        37.5\n",
            "19        4181         4119.783        61.2\n",
            "20        6765         6666.319        98.7\n",
            "21       10946        10786.473       159.5\n",
            "22       17711        17452.521       258.5\n",
            "23       28657        28239.209       417.8\n",
            "24       46368        45692.887       675.1\n",
            "25       75025        73933.367      1091.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_rzDtDj5gjA",
        "colab_type": "code",
        "outputId": "c017d799-4bd5-40a9-ca1e-61e7feb1e597",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        }
      },
      "source": [
        "sequence = fibonacci(start=(4, 5), num=25)\n",
        "predict_y = predict(sequence, best_model)\n",
        "mse = mean_squared_error(sequence, predict_y)\n",
        "print(f'mse is {mse}')\n",
        "plot_difference(sequence, predict_y, min_unit)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mse is 3093.770385513009\n",
            "Prediction of model with 50:\n",
            "    real value  predicted value  difference\n",
            "0            4            4.000         0.0\n",
            "1            5            5.000         0.0\n",
            "2            9            9.000         0.0\n",
            "3           14           14.000         0.0\n",
            "4           23           23.007         0.0\n",
            "5           37           37.017         0.0\n",
            "6           60           60.034         0.0\n",
            "7           97           97.038         0.0\n",
            "8          157          156.852         0.1\n",
            "9          254          253.927         0.1\n",
            "10         411          410.853         0.1\n",
            "11         665          664.877         0.1\n",
            "12        1076         1075.351         0.6\n",
            "13        1741         1740.231         0.8\n",
            "14        2817         2815.936         1.1\n",
            "15        4558         4555.651         2.3\n",
            "16        7375         7371.206         3.8\n",
            "17       11933        11927.281         5.7\n",
            "18       19308        19298.604         9.4\n",
            "19       31241        31225.939        15.1\n",
            "20       50549        50525.633        23.4\n",
            "21       81790        81753.312        36.7\n",
            "22      132339       132281.312        57.7\n",
            "23      214129       214039.188        89.8\n",
            "24      346468       346328.250       139.8\n",
            "25      560597       560379.625       217.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dNSxRbTM5lVD",
        "colab_type": "code",
        "outputId": "a476e5c8-de61-4dd9-8f60-6852d860240d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        }
      },
      "source": [
        "sequence = fibonacci(start=(3, 3), num=25)\n",
        "predict_y = predict(sequence, best_model)\n",
        "mse = mean_squared_error(sequence, predict_y)\n",
        "print(f'mse is {mse}')\n",
        "plot_difference(sequence, predict_y, min_unit)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mse is 2605.724642146611\n",
            "Prediction of model with 50:\n",
            "    real value  predicted value  difference\n",
            "0            3            3.000         0.0\n",
            "1            3            3.000         0.0\n",
            "2            6            6.000         0.0\n",
            "3            9            9.000         0.0\n",
            "4           15           14.972         0.0\n",
            "5           24           23.986         0.0\n",
            "6           39           39.057         0.1\n",
            "7           63           62.993         0.0\n",
            "8          102          101.936         0.1\n",
            "9          165          164.773         0.2\n",
            "10         267          267.051         0.1\n",
            "11         432          431.724         0.3\n",
            "12         699          698.600         0.4\n",
            "13        1131         1130.325         0.7\n",
            "14        1830         1828.954         1.0\n",
            "15        2961         2959.244         1.8\n",
            "16        4791         4787.868         3.1\n",
            "17        7752         7746.966         5.0\n",
            "18       12543        12534.983         8.0\n",
            "19       20295        20282.064        12.9\n",
            "20       32838        32817.402        20.6\n",
            "21       53133        53100.488        32.5\n",
            "22       85971        85919.562        51.4\n",
            "23      139104       139022.781        81.2\n",
            "24      225075       224947.172       127.8\n",
            "25      364179       363977.969       201.0\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}