Untitled93.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMZXeSN9z8Zk0lWx3OnSefc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rishabhRsinghvi/Animated-Data-Visualizations/blob/main/Untitled93.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B6r-xOWavl_r"
      },
      "outputs": [],
      "source": [
        "#CNN Digits:\n",
        "x_train /= 255\n",
        "x_test /=255\n",
        "batch_size=64\n",
        "num_classes=10\n",
        "epochs=2\n",
        "\n",
        "def build_model(optimizer):\n",
        "\n",
        "  model=Sequential()\n",
        "  model.add(Conv2D(32,kernel_size=(3,3),activation='relu',input_shape=input_shape))\n",
        "  model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "  model.add(Conv2D(32,kernel_size=(3,3),activation='relu'))\n",
        "  model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "  model.add(Dropout(0.25))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(256, activation='relu'))\n",
        "  model.add(Dropout(0.5))\n",
        "  model.add(Dense(num_classes, activation='softmax'))\n",
        "  model.compile(loss=keras.losses.categorical_crossentropy, optimizer= optimizer, metrics=['accuracy'])\n",
        "  model.summary()\n",
        "\n",
        "  return model\n",
        "\n",
        "# optimizers = ['Adadelta', 'Adagrad', 'Adam', 'RMSprop', 'SGD']\n",
        "# #for i in optimizers:\n",
        "\n",
        "model = build_model('Adam')\n",
        "\n",
        "def predict_image(model, img):\n",
        "  img = np.reshape(img,(1, 28, 28))\n",
        "  pred = model.predict(img)\n",
        "  img = img.astype('float32')\n",
        "  img /=255\n",
        "  print(pred)\n",
        "  answer = np.argmax(pred)\n",
        "  print(answer)\n",
        "\n",
        "\n",
        "#CNN fashion:\n",
        "x_train /= 255\n",
        "x_test /=255\n",
        "batch_size=64\n",
        "num_classes=10\n",
        "epochs=5\n",
        ".\n",
        ".\n",
        ".\n",
        "model_rmsprop = build_model(RMSprop())\n",
        "model_sgd = build_model(SGD())\n",
        "\n",
        "plot_model(model_rmsprop, to_file=\"mnist_model_rmsprop.jpg\", show_shapes=True)\n",
        "plot_model(model_sgd, to_file=\"mnist_model_sgd.jpg\", show_shapes=True)\n",
        "\n",
        "hist_rmsprop = model_rmsprop.fit(.......)\n",
        "hist_sgd = model_sgd.fit(........)\n",
        ".\n",
        ".\n",
        ".\n",
        "  img = np.reshape(img,(1, 28, 28))\n",
        "  pred = model.predict(img)\n",
        "  img = img.astype('float32')\n",
        "  img /=255\n",
        "  print(pred)\n",
        "  answer = np.argmax(pred)\n",
        "  print(labels[answer])\n",
        "\n",
        "\n",
        "\n",
        "#CNN Comparision:\n",
        "x_train /= 255\n",
        "x_test /=255\n",
        "batch_size=64\n",
        "\n",
        "num_classes=10\n",
        "epochs=1\n",
        "\n",
        "optimizers = ['Adadelta', 'Adagrad', 'Adam', 'RMSprop', 'SGD']\n",
        "for i in optimizers:\n",
        "  model = build_model(i)\n",
        "  plot_model(model, to_file=\"mnist model\"+ '.jpg', show_shapes=True)\n",
        "  hist=model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test,y_test))\n",
        "  y1=hist.history['accuracy']\n",
        "  y2=hist.history['val_accuracy']\n",
        "  plt.plot(y1)\n",
        "  plt.plot(y2)\n",
        "\n",
        "#LSTM:\n",
        "top_words = 5000\n",
        "(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words)\n",
        "\n",
        "max_review_length = 400\n",
        "X_train = pad_sequences(X_train, maxlen=max_review_length)\n",
        "X_test = pad_sequences(X_test, maxlen=max_review_length)\n",
        "\n",
        "\n",
        "embedding_vector_length = 32\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(top_words + 1, embedding_vector_length, input_length=max_review_length))\n",
        "model.add(LSTM(10))\n",
        "model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()\n",
        "\n",
        "print(\"Accuracy: %.2f%%\" % (scores[1]*100))\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "y1=hist.history['accuracy']          /  y1=hist.history['loss']\n",
        "y2=hist.history['val_accuracy']\n",
        "plt.plot(y1)\n",
        "plt.plot(y2)\n",
        "\n",
        "\n",
        "#Autoencoder\n",
        "# this is the size of our encoded representations\n",
        "input_dim = X_scaled.shape[1]\n",
        "encoding_dim = 2\n",
        "\n",
        "# this is our input placeholder\n",
        "input_img = Input(shape=(input_dim,))\n",
        "\n",
        "# \"encoded\" representation of the input\n",
        "encoded = Dense(encoding_dim,activation='sigmoid')(input_img)\n",
        "\n",
        "# \"decoded\" lossy reconstruction of the input\n",
        "decoded = Dense(input_dim,activation='sigmoid')(encoded)\n",
        "\n",
        "# Map an input to reconstruction\n",
        "autoencoder = Model(input_img,decoded)\n",
        "autoencoder.compile(optimizer='adam',loss='mse')\n",
        "print(autoencoder.summary())\n",
        "history = autoencoder.fit(X_scaled,X_scaled,epochs=2000,batch_size=16,shuffle=True,validation_split=0.1,verbose=0)\n",
        "\n",
        "\n",
        "\n",
        "#PCA:\n",
        "def plot3clusters(X, title, vtitle):\n",
        "    plt.figure()\n",
        "    colors = ['navy','turquoise','darkorange']\n",
        "    for color, i, target_name in zip(colors, [0,1,2], target_names):\n",
        "        plt.scatter(X[y==i, 0], X[y==i, 1], color=color, label=target_name)\n",
        "        plt.legend(loc='upper left')\n",
        "        plt.title(title)\n",
        "        plt.xlabel(vtitle + \"1\")\n",
        "        plt.ylabel(vtitle + \"2\")\n",
        "        plt.show()\n",
        "\n",
        "pca = PCA()\n",
        "pca_transformed = pca.fit_transform(X_scaled)\n",
        "\n",
        "# Displaying new Transformed Values\n",
        "print(\"Pca transformed: \", pca_transformed[0])\n",
        "\n",
        "# Calling the plotting function\n",
        "plot3clusters(pca_transformed[:,:2], 'PCA', 'PC')\n",
        "\n",
        "\n",
        "\n",
        "#LIME\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42) # create train and test\n",
        "\n",
        "# Building model - Xgboost\n",
        "model = XGBClassifier(random_state=42,gpu_id=0) # build classifier Gradient Boosted decision trees\n",
        "model.fit(X_train,y_train.values.ravel())\n",
        "\n",
        "# Classifier Function for lime explaination\n",
        "predict_fn = lambda x: model.predict_proba(x)\n",
        "\n",
        "explainer = lime.lime_tabular.LimeTabularExplainer(X_test.to_numpy(),\n",
        "                                            feature_names=data.feature_names,\n",
        "                                            class_names=['Negative','Positive'],\n",
        "                                            mode='classification',\n",
        "                                            verbose=True)\n",
        "#Explaining a single data point using lime explainer\n",
        "data_point = 5\n",
        "exp = explainer.explain_instance(df.loc[data_point,data.feature_names].astype(int).values, predict_fn, num_features=5)\n",
        "exp.show_in_notebook(show_table=True)\n",
        "\n",
        "figure = exp.as_pyplot_figure()\n",
        "print(*exp.available_labels())\n",
        "\n",
        "\n",
        "#SHAP:\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42) # create train and test\n",
        "model = XGBClassifier(random_state=42,gpu_id=0) # build classifier Gradient Boosted decision trees\n",
        "model.fit(X_train,y_train.values.ravel())\n",
        "\n",
        "explainer = shap.TreeExplainer(model)\n",
        "\n",
        "shap_values = explainer.shap_values(X)\n",
        "expected_value = explainer.expected_value\n",
        "\n",
        "\n",
        "#SOM:\n",
        "# Shape of the data:\n",
        "data.shape\n",
        "# Info of the data:\n",
        "data.info()\n",
        "# X variables:\n",
        "pd.DataFrame(X)\n",
        "\n",
        "sc = MinMaxScaler(feature_range = (0, 1))\n",
        "X = sc.fit_transform(X)\n",
        "pd.DataFrame(X)\n",
        "\n",
        "# define SOM:\n",
        "som = MiniSom(x = som_grid_rows, y = som_grid_columns, input_len=13, sigma=sigma, learning_rate=learning_rate)\n",
        "\n",
        "# Training\n",
        "som.train_random(X, iterations)\n",
        "\n",
        "\n",
        "mappings = som.win_map(X)\n",
        "mappings\n",
        "mappings.keys()\n",
        "len(mappings.keys())\n",
        "\n",
        "# the list of customers who are frauds:\n",
        "frauds1 = sc.inverse_transform(frauds)\n",
        "pd.DataFrame(frauds1)\n",
        "\n",
        "#GAN\n",
        "def define_discriminator(in_shape=(28, 28, 1)):\n",
        "    in_image = Input(shape=in_shape)\n",
        "    fe = Flatten()(in_image)\n",
        "    fe = Dense(1024)(fe)\n",
        "    fe = LeakyReLU(alpha=0.2)(fe)\n",
        "    fe = Dropout(0.3)(fe)\n",
        "    fe = Dense(512)(fe)\n",
        "    fe = LeakyReLU(alpha=0.2)(fe)\n",
        "    fe = Dropout(0.3)(fe)\n",
        "    fe = Dense(256)(fe)\n",
        "    fe = LeakyReLU(alpha=0.2)(fe)\n",
        "    fe = Dropout(0.3)(fe)\n",
        "    out = Dense(1, activation='sigmoid')(fe)\n",
        "    model = Model(in_image, out)\n",
        "    opt = Adam(lr=0.0002, beta_1=0.5)\n",
        "    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
        "    return model\n",
        "\n",
        "def define_generator(latent_dim):\n",
        "    init  = RandomNormal(stddev=0.02) #generates an array of specified shapes and fills it with random values, which is actually a part of Normal (Gaussian)Distribution\n",
        "    in_lat = Input(shape=(latent_dim,))\n",
        "    gen = Dense(256, kernel_initializer=init)(in_lat)\n",
        "    gen = LeakyReLU(alpha=0.2)(gen)\n",
        "    gen = Dense(512, kernel_initializer=init)(gen)\n",
        "    gen = LeakyReLU(alpha=0.2)(gen)\n",
        "    gen = Dense(1024, kernel_initializer=init)(gen)\n",
        "    gen = LeakyReLU(alpha=0.2)(gen)\n",
        "    gen = Dense(28 * 28 * 1, kernel_initializer=init)(gen)\n",
        "    out_layer = Activation('tanh')(gen)\n",
        "    out_layer = Reshape((28, 28, 1))(gen)\n",
        "    model = Model(in_lat, out_layer)\n",
        "    return model\n",
        "\n",
        "def define_gan(g_model, d_model):\n",
        "    d_model.trainable = False\n",
        "    gan_output = d_model(g_model.output)\n",
        "    model = Model(g_model.input, gan_output)\n",
        "    opt = Adam(lr=0.0002, beta_1=0.5)\n",
        "    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n",
        "    return model\n",
        "\n",
        "def train(g_model, d_model, gan_model, X_train, latent_dim, n_epochs=100, n_batch=64):\n",
        "    bat_per_epo = int(X_train.shape[0] / n_batch)\n",
        "    n_steps = bat_per_epo * n_epochs\n",
        "    for i in range(n_steps):\n",
        "        X_real, y_real = generate_real_samples(X_train, n_batch)\n",
        "        d_loss_r, d_acc_r = d_model.train_on_batch(X_real, y_real)\n",
        "        X_fake, y_fake = generate_fake_samples(g_model, latent_dim, n_batch)\n",
        "        d_loss_f, d_acc_f = d_model.train_on_batch(X_fake, y_fake)\n",
        "        z_input = generate_latent_points(latent_dim, n_batch)\n",
        "        y_gan = ones((n_batch, 1))\n",
        "        g_loss, g_acc = gan_model.train_on_batch(z_input, y_gan)\n",
        "        print('>%d, dr[%.3f,%.3f], df[%.3f,%.3f], g[%.3f,%.3f]' % (i+1, d_loss_r,d_acc_r, d_loss_f,d_acc_f, g_loss,g_acc))\n",
        "        if (i+1) % (bat_per_epo * 1) == 0:\n",
        "            summarize_performance(i, g_model, latent_dim)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#XOR\n",
        "import numpy as np\n",
        "def step_function(y_in):\n",
        "  if(y_in>=0):\n",
        "    return 1\n",
        "  else:\n",
        "    return 0\n",
        "\n",
        "def perceptron(x,w,b):\n",
        "  y_in=np.dot(x,w)+b\n",
        "  return y_in\n",
        "\n",
        "def or_function(x):\n",
        "  w=np.array([1,1])\n",
        "  b=-1\n",
        "  y_in=perceptron(x,w,b)\n",
        "  y_net=step_function(y_in)\n",
        "  return y_net\n",
        "\n",
        "def and_function(x):\n",
        "  w=np.array([1,1])\n",
        "  b=-2\n",
        "  y_in=perceptron(x,w,b)\n",
        "  y_net=step_function(y_in)\n",
        "  return y_net\n",
        "\n",
        "def not_function(x):\n",
        "  w=-1\n",
        "  b=0.5\n",
        "  y_in=perceptron(x,w,b)\n",
        "  y_net=step_function(y_in)\n",
        "  return y_net\n",
        "\n",
        "def xor_function(x):\n",
        "  y1=and_function(x)\n",
        "  y2=or_function(x)\n",
        "  y3=not_function(y1)\n",
        "  new_x=np.array([y2,y3])\n",
        "  return and_function(new_x)\n",
        "\n",
        "test1=np.array([0,0])\n",
        "test2=np.array([0,1])\n",
        "test3=np.array([1,0])\n",
        "test4=np.array([1,1])\n",
        "\n",
        "print(\"XOR GATE:\")\n",
        "print(\"\\nINPUT:\",test1,\"\\nOUTPUT:\",xor_function(test1))\n",
        "print(\"\\nINPUT:\",test2,\"\\nOUTPUT:\",xor_function(test2))\n",
        "print(\"\\nINPUT:\",test3,\"\\nOUTPUT:\",xor_function(test3))\n",
        "print(\"\\nINPUT:\",test4,\"\\nOUTPUT:\",xor_function(test4))\n",
        "\n",
        "#BackPropogation\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.datasets import load_iris\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "iris=load_iris()\n",
        "data=iris.data\n",
        "target=iris.target\n",
        "\n",
        "onehot_encoder=OneHotEncoder(sparse=False)\n",
        "reshaped=target.reshape(len(target),1)\n",
        "y_onehot=onehot_encoder.fit_transform(reshaped)\n",
        "iris_df=pd.DataFrame(data,columns=iris.feature_names)\n",
        "iris_df['target']=target\n",
        "\n",
        "X=iris_df.drop('target',axis=1)\n",
        "y=iris_df['target']\n",
        "\n",
        "X_train,X_test,y_train,y_test=train_test_split(X,y_onehot,test_size=0.2,random_state=42)\n",
        "\n",
        "input_size=4\n",
        "hidden_size=10\n",
        "output_size=3\n",
        "\n",
        "np.random.seed(0)\n",
        "\n",
        "V=np.random.rand(input_size,hidden_size)\n",
        "W=np.random.rand(hidden_size,output_size)\n",
        "\n",
        "def sigmoid(x):\n",
        "  return 1/(1+np.exp(-x))\n",
        "\n",
        "def error(y,y_net):\n",
        "  return np.mean((y-y_net)**2)\n",
        "\n",
        "def predict(X):\n",
        "  z_in=np.dot(X,V)\n",
        "  z=sigmoid(z_in)\n",
        "  y_in=np.dot(z,W)\n",
        "  y_net=sigmoid(y_in)\n",
        "  return np.argmax(y_net,axis=1)\n",
        "\n",
        "learning_rate=0.01\n",
        "mse=[]\n",
        "accuracy=[]\n",
        "iterations=1000\n",
        "\n",
        "for i in range(iterations):\n",
        "  z_in=np.dot(X_train,V)\n",
        "  z=sigmoid(z_in)\n",
        "  y_in=np.dot(z,W)\n",
        "  y_net=sigmoid(y_in)\n",
        "\n",
        "  mse.append(error(y_train,y_net))\n",
        "  y_pred=predict(X_test)\n",
        "  y_true=np.argmax(y_test,axis=1)\n",
        "  acc=np.mean(y_pred==y_true)\n",
        "  accuracy.append(acc)\n",
        "\n",
        "  dy=(y_train-y_net)*y_net*(1-y_net)\n",
        "  dw=np.dot(z.T,dy)\n",
        "  dz=np.dot(dy,W.T)*z*(1-z)\n",
        "  dv=np.dot(X_train.T,dz)\n",
        "\n",
        "  W=W+learning_rate*dw\n",
        "  V=V+learning_rate*dv\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "plt.figure(figsize=(12,4))\n",
        "\n",
        "plt.subplot(1,2,1)\n",
        "plt.plot(range(iterations),mse)\n",
        "plt.xlabel('Iterations')\n",
        "plt.ylabel('Mean Squared Error')\n",
        "plt.title('MSE V/S Iteretions')\n",
        "\n",
        "plt.subplot(1,2,2)\n",
        "plt.plot(range(iterations),accuracy)\n",
        "plt.xlabel('Iterations')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title('Accuracy V/S Iterations')\n",
        "\n",
        "plt.show()\n",
        "\n",
        "\n",
        "print(f\"The Final Accuracy on test data:{accuracy[-1]*100}%\")\n",
        "\n",
        "\n",
        "#Stochastic GD\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def sigmoid(y):\n",
        "  return 1/(1+np.exp(-y))\n",
        "\n",
        "def perceptron_model(x,w,b):\n",
        "  y_in=np.dot(x,w)+b\n",
        "  y=sigmoid(y_in)\n",
        "  return y\n",
        "\n",
        "def loss(y_true,y_pred):\n",
        "  return np.mean((y_true-y_pred)**2)\n",
        "\n",
        "def train_perceptron(X,Y,W,B,learning_rate,epochs):\n",
        "  for epoch in range(epochs):\n",
        "    total_loss=0\n",
        "    for i in range(len(X)):\n",
        "      x=X[i]\n",
        "      y=Y[i]\n",
        "\n",
        "      v=np.dot(x,W)+B\n",
        "      y_pred=perceptron_model(x,W,B)\n",
        "\n",
        "      dW=-2*(y-y_pred)*(y_pred)*(1-y_pred)*x\n",
        "      dB=-2*(y-y_pred)*(y_pred)*(1-y_pred)\n",
        "\n",
        "      Weight.append(dW)\n",
        "      Bias.append(dB)\n",
        "\n",
        "      W -=learning_rate*dW\n",
        "      B -=learning_rate*dB\n",
        "\n",
        "      total_loss += loss(y,y_pred)\n",
        "\n",
        "    avg_loss=total_loss/len(X)\n",
        "    loss_history.append(avg_loss)\n",
        "\n",
        "  return W, B\n",
        "\n",
        "X=np.array([[0.5,2.5]])\n",
        "Y=np.array([0.2,0.9])\n",
        "\n",
        "Weight=[]\n",
        "Bias=[]\n",
        "\n",
        "W=np.array([0.0,0.0])\n",
        "B=0.0\n",
        "\n",
        "learning_rate=0.1\n",
        "epochs=100\n",
        "\n",
        "loss_history=[]\n",
        "finalW,finalB=train_perceptron(X,Y,W,B,learning_rate,epochs)\n",
        "\n",
        "print(\"Final Weight:\",finalW)\n",
        "print(\"Final Bias:\",finalB)\n",
        "\n",
        "plt.plot(range(epochs), loss_history)\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Mean Squared Error Loss')\n",
        "plt.title('Loss Over Time')\n",
        "plt.show()\n",
        "\n",
        "\n",
        "plt.plot(Weight)\n",
        "\n",
        "\n",
        "#batch gradient descent\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "x = np.array([0.5, 2.5])\n",
        "y = np.array([0.2, 0.9])\n",
        "\n",
        "w = 0\n",
        "b = 0\n",
        "alpha = 0.1\n",
        "\n",
        "def batch_gradient_descent(x, y, w, b):\n",
        "    epoch = 10\n",
        "    weights_history = []\n",
        "    errors = []\n",
        "\n",
        "    for i in range(epoch):\n",
        "        dw = 0\n",
        "        db = 0\n",
        "        error_epoch = 0\n",
        "        for xi, yi in zip(x, y):\n",
        "            dw = dw + d_w(xi, yi, w, b, alpha)\n",
        "            db = db + d_b(xi, yi, w, b, alpha)\n",
        "            error_epoch += (yi - perceptron(xi, w, b)) ** 2\n",
        "\n",
        "        w = w + dw\n",
        "        b = b + db\n",
        "\n",
        "        weights_history.append((w, b))\n",
        "        errors.append(error_epoch / len(x))\n",
        "\n",
        "    print(\"The final weights are: \", w)\n",
        "    print(\"The final bias is: \", b)\n",
        "\n",
        "    return weights_history, errors\n",
        "\n",
        "def d_b(x, y, w, b, alpha):\n",
        "    y_hat = perceptron(x, w, b)\n",
        "    db = alpha * (y - y_hat) * y_hat * (1 - y_hat)\n",
        "    return db\n",
        "\n",
        "def d_w(x, y, w, b, alpha):\n",
        "    y_hat = perceptron(x, w, b)\n",
        "    dw = alpha * (y - y_hat) * y_hat * (1 - y_hat) * x\n",
        "    return dw\n",
        "\n",
        "def perceptron(x, w, b):\n",
        "    y_in = x * w + b\n",
        "    y_hat = sigmoid(y_in)\n",
        "    return y_hat\n",
        "\n",
        "def sigmoid(y_in):\n",
        "    y_hat = 1 / (1 + np.exp(-y_in))\n",
        "    return y_hat\n",
        "\n",
        "weights_history, errors = batch_gradient_descent(x, y, w, b)\n",
        "\n",
        "weight_0_history = [wh[0] for wh in weights_history]\n",
        "weight_1_history = [wh[1] for wh in weights_history]\n",
        "\n",
        "plt.figure(figsize=(10, 5))\n",
        "plt.subplot(121)\n",
        "plt.plot(weight_0_history)\n",
        "plt.title(\"Weight 0 Evolution\")\n",
        "plt.xlabel(\"Iteration\")\n",
        "plt.ylabel(\"Weight 0\")\n",
        "\n",
        "plt.subplot(122)\n",
        "plt.plot(weight_1_history)\n",
        "plt.title(\"Weight 1 Evolution\")\n",
        "plt.xlabel(\"Iteration\")\n",
        "plt.ylabel(\"Weight 1\")\n",
        "\n",
        "plt.tight_layout()\n",
        "\n",
        "plt.figure()\n",
        "plt.plot(errors)\n",
        "plt.title(\"Error Evolution\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Mean Squared Error\")\n",
        "plt.show()\n",
        "\n",
        "#minibatch\n",
        "def sigmoid(y_in):\n",
        "    y_hat = 1 / (1 + np.exp(-y_in))\n",
        "    return y_hat\n",
        "def perceptron(x, w, b):\n",
        "    y_in = x * w + b\n",
        "    y_hat = sigmoid(y_in)\n",
        "    return y_hat\n",
        "\n",
        "\n",
        "def grad_w(x, y, w, b):\n",
        "    y_hat = perceptron(x, w, b)\n",
        "    db =  (y - y_hat) * y_hat * (1 - y_hat)\n",
        "    return db\n",
        "\n",
        "def grad_b(x, y, w, b):\n",
        "    y_hat = perceptron(x, w, b)\n",
        "    dw =  (y - y_hat) * y_hat * (1 - y_hat) * x\n",
        "    return dw\n",
        "\n",
        "def loss(y_true, y_pred):\n",
        "    return (y_true - y_pred)**2\n",
        "\n",
        "\n",
        "def minibatch(w, b, x, y,a):\n",
        "    n = 0.1\n",
        "    epoch = 10\n",
        "    batch_size = int(input(\"Enter the batch size: \"))\n",
        "    for i in range(epoch):\n",
        "        dw, db, sample_no = 0, 0, 0\n",
        "        for xi, yi in zip(x, y):\n",
        "            dw += grad_w(w, b, xi, yi)\n",
        "            db += grad_b(w, b, xi, yi)\n",
        "            sample_no += 1\n",
        "            l.append(dw)\n",
        "            if sample_no % batch_size == 0:\n",
        "                w = w - dw*a\n",
        "                b = b - db *a\n",
        "\n",
        "    return w, b\n",
        "x=np.array([0.5,2.5])\n",
        "y=np.array([1.2,0.9])\n",
        "w=0.0\n",
        "b=0\n",
        "a=0.1\n",
        "l=[]\n",
        "new_w, new_b = minibatch(w, 0, x, y,a)\n",
        "\n",
        "print(\"Updated w:\", new_w)\n",
        "print(\"Updated b:\", new_b)\n",
        "\n",
        "\n",
        "##Batch momentum:\n",
        "def momentum_descent(w, b, x, y, alpha,beta, num_epochs):\n",
        "    v_w, v_b = 0.0, 0.0\n",
        "    for epoch in range(num_epochs):\n",
        "        dw,db=0,0\n",
        "        for xi, yi in zip(x, y):\n",
        "            dw = grad_w(w, b, xi, yi)\n",
        "            db = grad_b(w, b, xi, yi)\n",
        "\n",
        "        v_w = beta * v_w +(1-beta)* dw\n",
        "        v_b = beta * v_b + (1-beta)* db\n",
        "\n",
        "        w -= v_w*alpha\n",
        "        b -= v_b*alpha\n",
        "    return w, b\n",
        "\n",
        "x=np.array([0.5,2.5])\n",
        "y=np.array([1.2,0.9])\n",
        "w=0.0\n",
        "b=0\n",
        "a=0.1\n",
        "num_epochs = 10\n",
        "beta=0.9\n",
        "new_w, new_b = momentum_descent(w, 0.0, x, y, a, beta, num_epochs)\n",
        "\n",
        "print(\"Updated w:\", new_w)\n",
        "print(\"Updated b:\", new_b)\n",
        "\n",
        "\n",
        "#ADAGRAD:\n",
        "def adadelta(w, b, x, y, rho, epsilon, num_epochs):\n",
        "    w, b = 0.0, 0.0\n",
        "    E_dw, E_db = 0.0,0.0\n",
        "    delta_w, delta_b =  0.0,0.0\n",
        "\n",
        "    for epoch in range(num_epochs):\n",
        "        for xi, yi in zip(x, y):\n",
        "            dw = grad_w(w, b, xi, yi)\n",
        "            db = grad_b(w, b, xi, yi)\n",
        "\n",
        "            E_dw = rho * E_dw + (1 - rho) * (dw ** 2)\n",
        "            E_db = rho * E_db + (1 - rho) * (db ** 2)\n",
        "\n",
        "            delta_w = -np.sqrt(delta_w + epsilon) / (np.sqrt(E_dw + epsilon)) * dw\n",
        "            delta_b = -np.sqrt(delta_b + epsilon) / (np.sqrt(E_db + epsilon)) * db\n",
        "\n",
        "\n",
        "            w += delta_w\n",
        "            b += delta_b\n",
        "\n",
        "    return w, b\n",
        "\n",
        "x = np.array([0.5, 2.5])\n",
        "y = np.array([1.2, 0.9])\n",
        "w, b, alpha, rho, eps, num_epochs = 0.0, 0.0, 0.1, 0.95, 0.001, 10\n",
        "\n",
        "new_w, new_b = adadelta(w, b, x, y, rho, eps, num_epochs)\n",
        "print(\"Updated w:\", new_w)\n",
        "print(\"Updated b:\", new_b)\n",
        "\n",
        "def min_gradientdescent_adagrad(x, y, w, b, epoch, n, Batchsize, epsilon=0.00001):\n",
        "    grad_squared_w, grad_squared_b = 0, 0\n",
        "    for _ in range(epoch):\n",
        "        dw, db, sampleNo = 0, 0, 0\n",
        "        for xi, yi in zip(x, y):\n",
        "            y_pred = perceptronModel(xi, w, b)\n",
        "            dw += grad_W(w, b, xi, yi)\n",
        "            db += grad_B(w, b, xi, yi)\n",
        "            sampleNo += 1\n",
        "\n",
        "            if sampleNo % Batchsize == 0:\n",
        "\n",
        "                grad_squared_w += dw**2\n",
        "                grad_squared_b += db**2\n",
        "\n",
        "\n",
        "                ada_lr_w = n / (np.sqrt(grad_squared_w) + epsilon)\n",
        "                ada_lr_b = n / (np.sqrt(grad_squared_b) + epsilon)\n",
        "\n",
        "\n",
        "                w = w - ada_lr_w * dw\n",
        "                b = b - ada_lr_b * db\n",
        "\n",
        "\n",
        "                dw, db = 0, 0\n",
        "\n",
        "    return w, b\n",
        "\n",
        "\n",
        "##ADAM:\n",
        "import numpy as np\n",
        "def adam(w, b, x, y, alpha, beta1, beta2, epsilon, num_epochs):\n",
        "    w,b=0.0,0.0\n",
        "    m_w = np.zeros_like(w)\n",
        "    m_b, v_b = 0.0, 0.0\n",
        "    v_w = np.zeros_like(w)\n",
        "    i=0\n",
        "    for epoch in range(num_epochs):\n",
        "        for xi, yi in zip(x, y):\n",
        "            i += 1\n",
        "            dw = grad_w(w, b, xi, yi)\n",
        "            db = grad_b(w, b, xi, yi)\n",
        "            m_w = beta1 * m_w + (1 - beta1) * dw\n",
        "            m_b = beta1 * m_b + (1 - beta1) * db\n",
        "            v_w = beta2 * v_w + (1 - beta2) * (dw ** 2)\n",
        "            v_b = beta2 * v_b + (1 - beta2) * (db ** 2)\n",
        "            m_w_hat = m_w / (1 - beta1 ** i)\n",
        "            m_b_hat = m_b / (1 - beta1 ** i)\n",
        "            v_w_hat = v_w / (1 - beta2 ** i)\n",
        "            v_b_hat = v_b / (1 - beta2 ** i)\n",
        "            w -= (alpha / (np.sqrt(v_w_hat) + epsilon)) * m_w_hat\n",
        "            b -= (alpha / (np.sqrt(v_b_hat) + epsilon)) * m_b_hat\n",
        "    return w, b\n",
        "x=np.array([0.5,2.5])\n",
        "y=np.array([1.2,0.9])\n",
        "w,b,a=0.0,0,0.1\n",
        "num_epochs = 10\n",
        "esp=0.0001\n",
        "beta1,beta2 = 0.9,0.999\n",
        "new_w, new_b = adam(w, 0, x, y, a, beta1, beta2, eps, num_epochs)\n",
        "print(\"Updated w:\", new_w)\n",
        "print(\"Updated b:\", new_b)\n",
        "\n",
        "#NAG\n",
        "def NAG(w, b, x, y, alpha, beta, num_epochs):\n",
        "    v_w, v_b = 0.0, 0.0\n",
        "    for epoch in range(num_epochs):\n",
        "        for xi, yi in zip(x, y):\n",
        "            lookahead_dw = grad_w(w - beta * v_w, b - beta * v_b, xi, yi)\n",
        "            lookahead_db = grad_b(w - beta * v_w, b - beta * v_b, xi, yi)\n",
        "\n",
        "            v_w = beta * v_w - alpha * lookahead_dw\n",
        "            v_b = beta * v_b - alpha * lookahead_db\n",
        "\n",
        "            w += v_w\n",
        "            b += v_b\n",
        "\n",
        "    return w, b\n",
        "\n",
        "x=np.array([0.5,2.5])\n",
        "y=np.array([1.2,0.9])\n",
        "w=0.0\n",
        "b=0\n",
        "a=0.1\n",
        "num_epochs = 10\n",
        "beta=0.9\n",
        "new_w, new_b = NAG(w, 0, x, y, a, beta, num_epochs)\n",
        "\n",
        "print(\"Updated w:\", new_w)\n",
        "print(\"Updated b:\", new_b)\n"
      ],
      "metadata": {
        "id": "NegW3-C-wYYV"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}