Clustering_k-medoid_stream_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FHNQGRbeU3zj"
   },
   "source": [
    "**Clustream Algorithm**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wdFwszhgU3zl",
    "outputId": "09d69f5e-2579-42d3-c9f5-a9695a08304f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Type: <class 'numpy.ndarray'>\n",
      "Shape: (156, 9)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets, metrics\n",
    "\n",
    "\n",
    "#Reading data from a csv file\n",
    "from pandas import read_csv\n",
    "data=read_csv(filepath_or_buffer=\"newdiabetes1.csv\", header='infer')\n",
    "print(type(data))\n",
    "newdata=data.values\n",
    "print(\"Type:\",type(newdata))\n",
    "print(\"Shape:\",newdata.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UM865Q7BU3zq"
   },
   "source": [
    "Perform Scalling of the datasets using Standard Scaler "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HlBBcg6_U3zq"
   },
   "outputs": [],
   "source": [
    "#scale the data using standard scaler\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler=StandardScaler()\n",
    "scaler.fit(newdata)\n",
    "newdata=scaler.transform(newdata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ToMreH5CU3zu"
   },
   "source": [
    "# Perform necessary initialization of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vNu4Cc2wU3zu",
    "outputId": "41909ade-dd85-4a71-9331-7ea78e26172c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total no of data in datasets = 156\n"
     ]
    }
   ],
   "source": [
    "# To display number of the values in the datasets\n",
    "no_row= newdata.shape[0]\n",
    "print(\"Total no of data in datasets =\",no_row)\n",
    "\n",
    "chunk_size=6          \n",
    "chunk_limit=5*chunk_size \n",
    "k=2                        #  no of medians \n",
    "\n",
    "# Create single level matrix to store score of various levels of clusters . I assume that at max 10 levels will occurs \n",
    "level=np.zeros([10,100, 2],dtype=int)\n",
    "\n",
    "\n",
    "#create level_count matrix to store the count of weights in each chunk\n",
    "level_count=np.zeros(10,dtype=int)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "A9ZDQLm2U3zx"
   },
   "source": [
    "# User defined function to calculate number of clusters from Single Chunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tAxKAB_FU3zy"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "from numpy.random import choice\n",
    "from numpy.random import seed\n",
    "from pyclustering.cluster.kmedoids import kmedoids   \n",
    "from pyclustering.cluster.kmedians import kmedians\n",
    "\n",
    "\n",
    "def  mycluster(data1,k=2):\n",
    "\n",
    "    # Code for Random initialization of the Median for single chunk\n",
    "    seed(1)\n",
    "    samples=choice(len(data1),size=k,replace=False)\n",
    "    initial_medians=data1[samples,:]\n",
    "    # print(\"Initial Centroids\",initial_medians)\n",
    "    \n",
    "    \n",
    "    #code for the k-median cluster of single chunk\n",
    "    kmedians_instance = kmedians(data1, initial_medians)\n",
    "    # Run cluster analysis and obtain results.\n",
    "    kmedians_instance.process()\n",
    "    clusters1 = kmedians_instance.get_clusters()\n",
    "    return clusters1 \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u4i9_O7oU3z0"
   },
   "source": [
    "# Core logic of Stream algorithm. Find the cluseters of the data points and store & add the weightage of the clusters in the different level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9X36ZIroU3z0",
    "outputId": "99047e95-0613-4801-b700-2ac19e08cadd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Level no : 0\n",
      "Chunk No  0  : [4 2]\n",
      "Chunk No  1  : [4 2]\n",
      "Chunk No  2  : [5 1]\n",
      "Chunk No  3  : [4 2]\n",
      "Chunk No  4  : [1 5]\n",
      "Chunk No  5  : [4 2]\n",
      "Chunk No  6  : [2 4]\n",
      "Chunk No  7  : [1 5]\n",
      "Chunk No  8  : [4 2]\n",
      "Chunk No  9  : [4 2]\n",
      "Chunk No  10  : [3 3]\n",
      "Chunk No  11  : [4 2]\n",
      "Chunk No  12  : [1 5]\n",
      "Chunk No  13  : [1 5]\n",
      "Chunk No  14  : [4 2]\n",
      "Chunk No  15  : [1 5]\n",
      "Chunk No  16  : [2 4]\n",
      "Chunk No  17  : [4 2]\n",
      "Chunk No  18  : [1 5]\n",
      "Chunk No  19  : [4 2]\n",
      "Chunk No  20  : [1 5]\n",
      "Chunk No  21  : [2 4]\n",
      "Chunk No  22  : [2 4]\n",
      "Chunk No  23  : [2 4]\n",
      "Chunk No  24  : [5 1]\n",
      "Chunk No  25  : [1 4]\n",
      "\n",
      "Level no : 1\n",
      "Chunk No  0  : [13  5]\n",
      "Chunk No  1  : [9 9]\n",
      "Chunk No  2  : [ 7 11]\n",
      "Chunk No  3  : [11  7]\n",
      "Chunk No  4  : [ 6 12]\n",
      "Chunk No  5  : [ 7 11]\n",
      "Chunk No  6  : [ 6 12]\n",
      "Chunk No  7  : [ 6 12]\n",
      "\n",
      "Level no : 2\n",
      "Chunk No  0  : [29 25]\n",
      "Chunk No  1  : [24 30]\n",
      "\n",
      "No of dataframes at level  0 = 26\n",
      "No of dataframes at level  1 = 8\n",
      "No of dataframes at level  2 = 2\n",
      "Last Highest Level = 2\n",
      "\n",
      "FINAL CLUSTER WEIGHTAGE :  [[53 55]]\n"
     ]
    }
   ],
   "source": [
    "#pip install pyclustering\n",
    "\n",
    "\n",
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "from numpy.random import choice\n",
    "from numpy.random import seed\n",
    "from pyclustering.cluster.kmedians import kmedians\n",
    "\n",
    "\n",
    "#print(level)\n",
    "\n",
    "chunk_no=0\n",
    "\n",
    "# call the forloop to iterate over all the data with the increament of chunk_size\n",
    "for i in range(1,no_row,chunk_size):\n",
    "    \n",
    "    # store single chunk in data1\n",
    "    data1=newdata[i:i+chunk_size]\n",
    "    # print(\"data of chunk\" ,data1)\n",
    "    #print(data1.shape)  \n",
    "    \n",
    "    \n",
    "    # Call myCluster function to find the cluster for single data clusters\n",
    "    clusters=mycluster(data1,2)\n",
    "    \n",
    "    #store the count of each cluster in level1 matrix \n",
    "    cluslength=len(clusters)\n",
    "    level_number=0\n",
    "    clusterno=0\n",
    "    \n",
    "    # Update the cluster weightage in the first level for Single data cluster \n",
    "    for i in clusters:\n",
    "        level[0][chunk_no][clusterno]=len(i)\n",
    "       # level[level_number][chunk_no][clusterno]=len(i)\n",
    "        clusterno=clusterno+1\n",
    "  \n",
    "    chunk_no=chunk_no+1\n",
    "   \n",
    "    level_count[0]=level_count[0] + 1\n",
    "    first_level_count=level_count[0]\n",
    "     \n",
    "    # Code to check if dataframe of any level has exceed the size or not\n",
    "    if ((first_level_count * k) % chunk_size) == 0: \n",
    "        current_level=level_number  \n",
    "        x=1 \n",
    "        \n",
    "        # Iterate while loop continuously until we update necessary level's data\n",
    "        while x==1:\n",
    "            \n",
    "            next_level=current_level + 1\n",
    "            next_level_chunk_no=level_count[next_level]\n",
    "            \n",
    "            # count the weight of different clusters in the level matrix            \n",
    "            for p in range(k):\n",
    "                 \n",
    "                l=level_count[current_level]\n",
    "                k1_count=level[current_level][l-1][p]+level[current_level][l-2][p]+level[current_level][l-3][p]\n",
    "      \n",
    "                level[next_level][next_level_chunk_no][p]=k1_count\n",
    "            \n",
    "            level_count[next_level]= next_level_chunk_no + 1\n",
    "            \n",
    "            if ((level_count[next_level])* k) % chunk_size == 0  :\n",
    "                current_level=next_level\n",
    "                next_level=next_level +1\n",
    "                \n",
    "            else :\n",
    "                 x=2\n",
    "\n",
    "    \n",
    "# Print the number of cluster weightage per data chunk at each level\n",
    "for i in range(10):\n",
    "    if level_count[i] > 0 : \n",
    "        print()\n",
    "        print(\"Level no :\",i)\n",
    "        l=level_count[i]\n",
    "        for j in range(l):\n",
    "            print(\"Chunk No \",j,\" :\",level[i][j])\n",
    "\n",
    "print()            \n",
    "# Print number of dataframes at individual level\n",
    "for j in range(10):\n",
    "    if level_count[j] > 0 :\n",
    "        max_level=j\n",
    "        print(\"No of dataframes at level \",j,\"=\",level_count[j])\n",
    "\n",
    "print(\"Last Highest Level =\",max_level)\n",
    "\n",
    "\n",
    "# create 2 dimensional matrix to store final cluster weightage count\n",
    "final_cluster=np.zeros([1,k],dtype=int)\n",
    "\n",
    "# Check if maximul level has number of elements less than chunk size then add all the elements for final summation\n",
    "if (level_count[max_level] * k) % chunk_size != 0  :\n",
    "    \n",
    "    # Find the final level\n",
    "    m=level_count[max_level] % chunk_size \n",
    "        \n",
    "    # Iterate over all the k-clusters of maximuk level to find sum\n",
    "    for i in range(k):\n",
    "       # print(\"i=\",i)\n",
    "       # print(\"length of i =\",len(i))\n",
    "        for j in range(m):\n",
    "            final_cluster[0][i]=final_cluster[0][i]+level[max_level][j][i]\n",
    "\n",
    "\n",
    "# print final cluster weightage of the data\n",
    "print()\n",
    "print(\"FINAL CLUSTER WEIGHTAGE : \",final_cluster)\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "AOBD_Assignment_9.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}