Process_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "010a7ac1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy.random import randint\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ed294235",
   "metadata": {},
   "outputs": [],
   "source": [
    "from plotting.data.databreach2 import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "17d55c2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num data points:  540\n",
      "columns:  dict_keys(['Name', 'Title', 'Domain', 'BreachDate', 'AddedDate', 'ModifiedDate', 'PwnCount', 'Description', 'LogoPath', 'DataClasses', 'IsVerified', 'IsFabricated', 'IsSensitive', 'IsRetired', 'IsSpamList'])\n"
     ]
    }
   ],
   "source": [
    "print(\"num data points: \", len(data))\n",
    "print(\"columns: \", data[0].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "10f0d595",
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = pd.read_csv(\"./plotting/data/databreach.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "02f0c487",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Org</th>\n",
       "      <th>Category</th>\n",
       "      <th>breach_count</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10-2017</td>\n",
       "      <td>Yahoo</td>\n",
       "      <td>web services</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>[ Security questions, security answers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03-2018</td>\n",
       "      <td>UIDAI</td>\n",
       "      <td>AADHAR</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>[ Names, Aadhar number, bank details]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>04-2019</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>social media</td>\n",
       "      <td>540.0</td>\n",
       "      <td>[comments, likes, reactions, account names, FB...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014</td>\n",
       "      <td>Yahoo</td>\n",
       "      <td>web services</td>\n",
       "      <td>500.0</td>\n",
       "      <td>[names, email addresses, phone numbers, hashed...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>06-2013</td>\n",
       "      <td>MySpace</td>\n",
       "      <td>social media</td>\n",
       "      <td>360.0</td>\n",
       "      <td>[owner’s listed name, username, birthdate]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>06-2018</td>\n",
       "      <td>Exactis</td>\n",
       "      <td>web services</td>\n",
       "      <td>340.0</td>\n",
       "      <td>[phone numbers, home, and email addresses, int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>05-2018</td>\n",
       "      <td>Twitter</td>\n",
       "      <td>social media</td>\n",
       "      <td>330.0</td>\n",
       "      <td>[password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10-2015</td>\n",
       "      <td>NetEase</td>\n",
       "      <td>web services</td>\n",
       "      <td>234.0</td>\n",
       "      <td>[email address, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>06-2017</td>\n",
       "      <td>Deep Root Analytics</td>\n",
       "      <td>web services</td>\n",
       "      <td>200.0</td>\n",
       "      <td>[voter Personal Identifiable Information (PII)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12-2018</td>\n",
       "      <td>Dubsmash</td>\n",
       "      <td>social media</td>\n",
       "      <td>162.0</td>\n",
       "      <td>[email addresses, usernames, DBKDF2 password h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10-2013</td>\n",
       "      <td>Adobe</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>152.0</td>\n",
       "      <td>[internal ID, username, email, encrypted passw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>02-2018</td>\n",
       "      <td>MyFitnessPal</td>\n",
       "      <td>health</td>\n",
       "      <td>150.0</td>\n",
       "      <td>[email addresses, IP addresses, usernames, pas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>09-2017</td>\n",
       "      <td>Equifax</td>\n",
       "      <td>web services</td>\n",
       "      <td>148.0</td>\n",
       "      <td>[names, home addresses, phone numbers, dates o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>03-2014</td>\n",
       "      <td>ebay</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>145.0</td>\n",
       "      <td>[names, e-mail addresses, physical addresses, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>05-2019</td>\n",
       "      <td>Canva</td>\n",
       "      <td>web services</td>\n",
       "      <td>137.0</td>\n",
       "      <td>[email addresses, names, usernames, cities, pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>06-2018</td>\n",
       "      <td>Apollo</td>\n",
       "      <td>health</td>\n",
       "      <td>126.0</td>\n",
       "      <td>[names, email addresses, place of employment, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>06-2013</td>\n",
       "      <td>Badoo</td>\n",
       "      <td>web services</td>\n",
       "      <td>112.0</td>\n",
       "      <td>[names, birthdates, passwords, email addresses]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>08-2013</td>\n",
       "      <td>Evite</td>\n",
       "      <td>web services</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[email addresses, phone numbers, names, physic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12-2018</td>\n",
       "      <td>Quora</td>\n",
       "      <td>social media</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[names, email addresses, encrypted passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>12-2016</td>\n",
       "      <td>Youku</td>\n",
       "      <td>web services</td>\n",
       "      <td>92.0</td>\n",
       "      <td>[usernames, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>03-2014</td>\n",
       "      <td>Rambler</td>\n",
       "      <td>search engine</td>\n",
       "      <td>91.0</td>\n",
       "      <td>[usernames, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>10-2016</td>\n",
       "      <td>Dailymotion</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>85.0</td>\n",
       "      <td>[emails, username,passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2012</td>\n",
       "      <td>Dropbox</td>\n",
       "      <td>web services</td>\n",
       "      <td>69.0</td>\n",
       "      <td>[email addresses, passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>02-2013</td>\n",
       "      <td>tumblr</td>\n",
       "      <td>social media</td>\n",
       "      <td>66.0</td>\n",
       "      <td>[email addresses, passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2016</td>\n",
       "      <td>Uber</td>\n",
       "      <td>transport</td>\n",
       "      <td>57.0</td>\n",
       "      <td>[names, email addresses, mobile phone numbers,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>01-2021</td>\n",
       "      <td>Bonobos</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>12.3</td>\n",
       "      <td>[shipping address records, account information...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>01-2021</td>\n",
       "      <td>Pixlr</td>\n",
       "      <td>web services</td>\n",
       "      <td>1.9</td>\n",
       "      <td>[Usernames, Email addresses, Country, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>04-2020</td>\n",
       "      <td>Zoom</td>\n",
       "      <td>web services</td>\n",
       "      <td>0.5</td>\n",
       "      <td>[login credentials]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>04-2020</td>\n",
       "      <td>Nintendo</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>0.3</td>\n",
       "      <td>[Account passwords, Account owner name, DOB, E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>09-2020</td>\n",
       "      <td>Mailfire</td>\n",
       "      <td>web services</td>\n",
       "      <td>0.1</td>\n",
       "      <td>[Name, Age, DOB, Gender, Location of message s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>10-2016</td>\n",
       "      <td>Hitachi Payment Services System</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>3.2</td>\n",
       "      <td>[debit card number, PIN]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>01-2019</td>\n",
       "      <td>SBI</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>3.5</td>\n",
       "      <td>[mobile numbers, partial account numbers, acco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>04-2019</td>\n",
       "      <td>Justdial</td>\n",
       "      <td>web services</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[names, mobile numbers, email ids, occupations...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>10-2019</td>\n",
       "      <td>Multiple Indian Banks</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>1.3</td>\n",
       "      <td>[names, emails, phone numbers, addresses]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>11-2020</td>\n",
       "      <td>BigBasket</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>20.0</td>\n",
       "      <td>[email id, phone numbers, order details, addre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>05-2020</td>\n",
       "      <td>Unacademy</td>\n",
       "      <td>education</td>\n",
       "      <td>20.0</td>\n",
       "      <td>[IDs, passwords, date joined, last login date,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>05-2021</td>\n",
       "      <td>Air India</td>\n",
       "      <td>airlines</td>\n",
       "      <td>4.5</td>\n",
       "      <td>[passport, credit card details, birth dates, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>05-2021</td>\n",
       "      <td>Dominos</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>180.0</td>\n",
       "      <td>[order details, email addresses, phone numbers...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Date                              Org       Category  breach_count  \\\n",
       "0   10-2017                            Yahoo   web services        3000.0   \n",
       "1   03-2018                            UIDAI         AADHAR        1100.0   \n",
       "2   04-2019                         Facebook   social media         540.0   \n",
       "3      2014                            Yahoo   web services         500.0   \n",
       "4   06-2013                          MySpace   social media         360.0   \n",
       "5   06-2018                          Exactis   web services         340.0   \n",
       "6   05-2018                          Twitter   social media         330.0   \n",
       "7   10-2015                          NetEase   web services         234.0   \n",
       "8   06-2017              Deep Root Analytics   web services         200.0   \n",
       "9   12-2018                         Dubsmash   social media         162.0   \n",
       "10  10-2013                            Adobe     e commerce         152.0   \n",
       "11  02-2018                     MyFitnessPal         health         150.0   \n",
       "12  09-2017                          Equifax   web services         148.0   \n",
       "13  03-2014                             ebay     e commerce         145.0   \n",
       "14  05-2019                            Canva   web services         137.0   \n",
       "15  06-2018                           Apollo         health         126.0   \n",
       "16  06-2013                            Badoo   web services         112.0   \n",
       "17  08-2013                            Evite   web services         101.0   \n",
       "18  12-2018                            Quora   social media         100.0   \n",
       "19  12-2016                            Youku   web services          92.0   \n",
       "20  03-2014                          Rambler  search engine          91.0   \n",
       "21  10-2016                      Dailymotion  entertainment          85.0   \n",
       "22     2012                          Dropbox   web services          69.0   \n",
       "23  02-2013                           tumblr   social media          66.0   \n",
       "24     2016                             Uber      transport          57.0   \n",
       "25  01-2021                          Bonobos     e commerce          12.3   \n",
       "26  01-2021                            Pixlr   web services           1.9   \n",
       "27  04-2020                             Zoom   web services           0.5   \n",
       "28  04-2020                         Nintendo  entertainment           0.3   \n",
       "29  09-2020                         Mailfire   web services           0.1   \n",
       "30  10-2016  Hitachi Payment Services System     e commerce           3.2   \n",
       "31  01-2019                              SBI     e commerce           3.5   \n",
       "32  04-2019                         Justdial   web services         100.0   \n",
       "33  10-2019            Multiple Indian Banks     e commerce           1.3   \n",
       "34  11-2020                        BigBasket     e commerce          20.0   \n",
       "35  05-2020                        Unacademy      education          20.0   \n",
       "36  05-2021                        Air India       airlines           4.5   \n",
       "37  05-2021                          Dominos     e commerce         180.0   \n",
       "\n",
       "                                                 type  \n",
       "0             [ Security questions, security answers]  \n",
       "1               [ Names, Aadhar number, bank details]  \n",
       "2   [comments, likes, reactions, account names, FB...  \n",
       "3   [names, email addresses, phone numbers, hashed...  \n",
       "4          [owner’s listed name, username, birthdate]  \n",
       "5   [phone numbers, home, and email addresses, int...  \n",
       "6                                          [password]  \n",
       "7                           [email address, password]  \n",
       "8   [voter Personal Identifiable Information (PII)...  \n",
       "9   [email addresses, usernames, DBKDF2 password h...  \n",
       "10  [internal ID, username, email, encrypted passw...  \n",
       "11  [email addresses, IP addresses, usernames, pas...  \n",
       "12  [names, home addresses, phone numbers, dates o...  \n",
       "13  [names, e-mail addresses, physical addresses, ...  \n",
       "14  [email addresses, names, usernames, cities, pa...  \n",
       "15  [names, email addresses, place of employment, ...  \n",
       "16    [names, birthdates, passwords, email addresses]  \n",
       "17  [email addresses, phone numbers, names, physic...  \n",
       "18      [names, email addresses, encrypted passwords]  \n",
       "19                              [usernames, password]  \n",
       "20                              [usernames, password]  \n",
       "21                       [emails, username,passwords]  \n",
       "22                       [email addresses, passwords]  \n",
       "23                       [email addresses, passwords]  \n",
       "24  [names, email addresses, mobile phone numbers,...  \n",
       "25  [shipping address records, account information...  \n",
       "26    [Usernames, Email addresses, Country, password]  \n",
       "27                                [login credentials]  \n",
       "28  [Account passwords, Account owner name, DOB, E...  \n",
       "29  [Name, Age, DOB, Gender, Location of message s...  \n",
       "30                           [debit card number, PIN]  \n",
       "31  [mobile numbers, partial account numbers, acco...  \n",
       "32  [names, mobile numbers, email ids, occupations...  \n",
       "33          [names, emails, phone numbers, addresses]  \n",
       "34  [email id, phone numbers, order details, addre...  \n",
       "35  [IDs, passwords, date joined, last login date,...  \n",
       "36  [passport, credit card details, birth dates, n...  \n",
       "37  [order details, email addresses, phone numbers...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f8eae71a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data2.type[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "096161c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "Names = [ i[\"Title\"] for i in data ]\n",
    "dates = [ i[\"BreachDate\"] for i in data ]\n",
    "cat = [ None for i in data]\n",
    "feilds = [ i[\"DataClasses\"] for i in data ]\n",
    "count = [ i[\"PwnCount\"] for i in data ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d5ac5da1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, val in enumerate(data2[\"Date\"]):\n",
    "    Names.append(data2[\"Org\"][idx])\n",
    "    dates.append(data2[\"Date\"][idx])\n",
    "    cat.append(data2[\"Category\"][idx])\n",
    "#     print(data2[\"type\"][idx].split(\",\").trim())\n",
    "    feilds.append(data2[\"type\"][idx].split(\",\"))\n",
    "    count.append(data2[\"breach_count\"][idx]*1000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "01f42a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Df_data = pd.DataFrame([Names, dates, cat, feilds, count])\n",
    "Df_data = Df_data.T\n",
    "Df_data.columns=[\"Org\", \"Breach Date\", \"Category\", \"Data Feilds\", \"People Impacted\"] "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22e3912f",
   "metadata": {},
   "source": [
    "### Sample data for company size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eeb5ffdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SIZE = 1e9\n",
    "Df_data['Org Size'] = randint(MAX_SIZE, size=len(Df_data['Org']), dtype=int)\n",
    "\n",
    "# temp = []\n",
    "# for i in range(len(Df_data['Org'])):\n",
    "#     temp.append(int(randint(MAX_SIZE)))\n",
    "# Df_data['Org Size'] = temp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9492f9bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(Df_data['Org Size'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e39d426c",
   "metadata": {},
   "source": [
    "## Make dict and Save data as json\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "11858c26",
   "metadata": {},
   "source": [
    "demo: {\n",
    "      labels: [\"Label 1\", \"Label 2\", \"Label 3\"],\n",
    "      datasets: [{\n",
    "         label: 'Legend',\n",
    "         data: [{\n",
    "            x: -10,\n",
    "            y: 0,\n",
    "         }, {\n",
    "            x: 0,\n",
    "            y: 10\n",
    "         }, {\n",
    "            x: 10,\n",
    "            y: 5\n",
    "         }]\n",
    "      }]\n",
    "   }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3e2c90f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "final = { \"labels\": [], \n",
    "         \"datasets\":[{\"backgroundColor\": \"purple\",\n",
    "                      \"pointBackgroundColor\": \"#80008069\",\n",
    "                      \"pointHoverBackgroundColor\": \"red\",\n",
    "                      \"pointHoverRadius\": \"5\",\n",
    "                      \"label\": 'Legend',\n",
    "                      \"data\": []}] }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5452e402",
   "metadata": {},
   "outputs": [],
   "source": [
    "for num, i in enumerate(Df_data['Org']):\n",
    "    final['labels'].append(i)\n",
    "    final['datasets'][0]['data'].append({\n",
    "                                \"x\": int(Df_data['Org Size'][num]),\n",
    "                                \"y\": int(len(Df_data['Data Feilds'][num])*Df_data['People Impacted'][num])\n",
    "                                })"
   ]
  },
  {
   "cell_type": "raw",
   "id": "ae6d35a2",
   "metadata": {},
   "source": [
    "out_file = open(\"./plotting/data/plotting_file.json\", \"w\") \n",
    "    \n",
    "json.dump(final, out_file, indent = 4) \n",
    "    \n",
    "out_file.close() "
   ]
  },
  {
   "cell_type": "raw",
   "id": "bf386150",
   "metadata": {},
   "source": [
    "Df_data[\"People Impacted\"].hist()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "3939b6e2",
   "metadata": {},
   "source": [
    "Df_data[Df_data[\"People Impacted\"] < 1e9].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e2102dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f71437cd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "7874d076",
   "metadata": {},
   "source": [
    "## Data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e3d51ef4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Org</th>\n",
       "      <th>Category</th>\n",
       "      <th>breach_count</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10-2017</td>\n",
       "      <td>Yahoo</td>\n",
       "      <td>web services</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>[ Security questions, security answers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03-2018</td>\n",
       "      <td>UIDAI</td>\n",
       "      <td>AADHAR</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>[ Names, Aadhar number, bank details]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>04-2019</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>social media</td>\n",
       "      <td>540.0</td>\n",
       "      <td>[comments, likes, reactions, account names, FB...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014</td>\n",
       "      <td>Yahoo</td>\n",
       "      <td>web services</td>\n",
       "      <td>500.0</td>\n",
       "      <td>[names, email addresses, phone numbers, hashed...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>06-2013</td>\n",
       "      <td>MySpace</td>\n",
       "      <td>social media</td>\n",
       "      <td>360.0</td>\n",
       "      <td>[owner’s listed name, username, birthdate]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>06-2018</td>\n",
       "      <td>Exactis</td>\n",
       "      <td>web services</td>\n",
       "      <td>340.0</td>\n",
       "      <td>[phone numbers, home, and email addresses, int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>05-2018</td>\n",
       "      <td>Twitter</td>\n",
       "      <td>social media</td>\n",
       "      <td>330.0</td>\n",
       "      <td>[password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10-2015</td>\n",
       "      <td>NetEase</td>\n",
       "      <td>web services</td>\n",
       "      <td>234.0</td>\n",
       "      <td>[email address, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>06-2017</td>\n",
       "      <td>Deep Root Analytics</td>\n",
       "      <td>web services</td>\n",
       "      <td>200.0</td>\n",
       "      <td>[voter Personal Identifiable Information (PII)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12-2018</td>\n",
       "      <td>Dubsmash</td>\n",
       "      <td>social media</td>\n",
       "      <td>162.0</td>\n",
       "      <td>[email addresses, usernames, DBKDF2 password h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10-2013</td>\n",
       "      <td>Adobe</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>152.0</td>\n",
       "      <td>[internal ID, username, email, encrypted passw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>02-2018</td>\n",
       "      <td>MyFitnessPal</td>\n",
       "      <td>health</td>\n",
       "      <td>150.0</td>\n",
       "      <td>[email addresses, IP addresses, usernames, pas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>09-2017</td>\n",
       "      <td>Equifax</td>\n",
       "      <td>web services</td>\n",
       "      <td>148.0</td>\n",
       "      <td>[names, home addresses, phone numbers, dates o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>03-2014</td>\n",
       "      <td>ebay</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>145.0</td>\n",
       "      <td>[names, e-mail addresses, physical addresses, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>05-2019</td>\n",
       "      <td>Canva</td>\n",
       "      <td>web services</td>\n",
       "      <td>137.0</td>\n",
       "      <td>[email addresses, names, usernames, cities, pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>06-2018</td>\n",
       "      <td>Apollo</td>\n",
       "      <td>health</td>\n",
       "      <td>126.0</td>\n",
       "      <td>[names, email addresses, place of employment, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>06-2013</td>\n",
       "      <td>Badoo</td>\n",
       "      <td>web services</td>\n",
       "      <td>112.0</td>\n",
       "      <td>[names, birthdates, passwords, email addresses]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>08-2013</td>\n",
       "      <td>Evite</td>\n",
       "      <td>web services</td>\n",
       "      <td>101.0</td>\n",
       "      <td>[email addresses, phone numbers, names, physic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12-2018</td>\n",
       "      <td>Quora</td>\n",
       "      <td>social media</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[names, email addresses, encrypted passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>12-2016</td>\n",
       "      <td>Youku</td>\n",
       "      <td>web services</td>\n",
       "      <td>92.0</td>\n",
       "      <td>[usernames, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>03-2014</td>\n",
       "      <td>Rambler</td>\n",
       "      <td>search engine</td>\n",
       "      <td>91.0</td>\n",
       "      <td>[usernames, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>10-2016</td>\n",
       "      <td>Dailymotion</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>85.0</td>\n",
       "      <td>[emails, username,passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2012</td>\n",
       "      <td>Dropbox</td>\n",
       "      <td>web services</td>\n",
       "      <td>69.0</td>\n",
       "      <td>[email addresses, passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>02-2013</td>\n",
       "      <td>tumblr</td>\n",
       "      <td>social media</td>\n",
       "      <td>66.0</td>\n",
       "      <td>[email addresses, passwords]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2016</td>\n",
       "      <td>Uber</td>\n",
       "      <td>transport</td>\n",
       "      <td>57.0</td>\n",
       "      <td>[names, email addresses, mobile phone numbers,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>01-2021</td>\n",
       "      <td>Bonobos</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>12.3</td>\n",
       "      <td>[shipping address records, account information...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>01-2021</td>\n",
       "      <td>Pixlr</td>\n",
       "      <td>web services</td>\n",
       "      <td>1.9</td>\n",
       "      <td>[Usernames, Email addresses, Country, password]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>04-2020</td>\n",
       "      <td>Zoom</td>\n",
       "      <td>web services</td>\n",
       "      <td>0.5</td>\n",
       "      <td>[login credentials]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>04-2020</td>\n",
       "      <td>Nintendo</td>\n",
       "      <td>entertainment</td>\n",
       "      <td>0.3</td>\n",
       "      <td>[Account passwords, Account owner name, DOB, E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>09-2020</td>\n",
       "      <td>Mailfire</td>\n",
       "      <td>web services</td>\n",
       "      <td>0.1</td>\n",
       "      <td>[Name, Age, DOB, Gender, Location of message s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>10-2016</td>\n",
       "      <td>Hitachi Payment Services System</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>3.2</td>\n",
       "      <td>[debit card number, PIN]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>01-2019</td>\n",
       "      <td>SBI</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>3.5</td>\n",
       "      <td>[mobile numbers, partial account numbers, acco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>04-2019</td>\n",
       "      <td>Justdial</td>\n",
       "      <td>web services</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[names, mobile numbers, email ids, occupations...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>10-2019</td>\n",
       "      <td>Multiple Indian Banks</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>1.3</td>\n",
       "      <td>[names, emails, phone numbers, addresses]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>11-2020</td>\n",
       "      <td>BigBasket</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>20.0</td>\n",
       "      <td>[email id, phone numbers, order details, addre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>05-2020</td>\n",
       "      <td>Unacademy</td>\n",
       "      <td>education</td>\n",
       "      <td>20.0</td>\n",
       "      <td>[IDs, passwords, date joined, last login date,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>05-2021</td>\n",
       "      <td>Air India</td>\n",
       "      <td>airlines</td>\n",
       "      <td>4.5</td>\n",
       "      <td>[passport, credit card details, birth dates, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>05-2021</td>\n",
       "      <td>Dominos</td>\n",
       "      <td>e commerce</td>\n",
       "      <td>180.0</td>\n",
       "      <td>[order details, email addresses, phone numbers...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Date                              Org       Category  breach_count  \\\n",
       "0   10-2017                            Yahoo   web services        3000.0   \n",
       "1   03-2018                            UIDAI         AADHAR        1100.0   \n",
       "2   04-2019                         Facebook   social media         540.0   \n",
       "3      2014                            Yahoo   web services         500.0   \n",
       "4   06-2013                          MySpace   social media         360.0   \n",
       "5   06-2018                          Exactis   web services         340.0   \n",
       "6   05-2018                          Twitter   social media         330.0   \n",
       "7   10-2015                          NetEase   web services         234.0   \n",
       "8   06-2017              Deep Root Analytics   web services         200.0   \n",
       "9   12-2018                         Dubsmash   social media         162.0   \n",
       "10  10-2013                            Adobe     e commerce         152.0   \n",
       "11  02-2018                     MyFitnessPal         health         150.0   \n",
       "12  09-2017                          Equifax   web services         148.0   \n",
       "13  03-2014                             ebay     e commerce         145.0   \n",
       "14  05-2019                            Canva   web services         137.0   \n",
       "15  06-2018                           Apollo         health         126.0   \n",
       "16  06-2013                            Badoo   web services         112.0   \n",
       "17  08-2013                            Evite   web services         101.0   \n",
       "18  12-2018                            Quora   social media         100.0   \n",
       "19  12-2016                            Youku   web services          92.0   \n",
       "20  03-2014                          Rambler  search engine          91.0   \n",
       "21  10-2016                      Dailymotion  entertainment          85.0   \n",
       "22     2012                          Dropbox   web services          69.0   \n",
       "23  02-2013                           tumblr   social media          66.0   \n",
       "24     2016                             Uber      transport          57.0   \n",
       "25  01-2021                          Bonobos     e commerce          12.3   \n",
       "26  01-2021                            Pixlr   web services           1.9   \n",
       "27  04-2020                             Zoom   web services           0.5   \n",
       "28  04-2020                         Nintendo  entertainment           0.3   \n",
       "29  09-2020                         Mailfire   web services           0.1   \n",
       "30  10-2016  Hitachi Payment Services System     e commerce           3.2   \n",
       "31  01-2019                              SBI     e commerce           3.5   \n",
       "32  04-2019                         Justdial   web services         100.0   \n",
       "33  10-2019            Multiple Indian Banks     e commerce           1.3   \n",
       "34  11-2020                        BigBasket     e commerce          20.0   \n",
       "35  05-2020                        Unacademy      education          20.0   \n",
       "36  05-2021                        Air India       airlines           4.5   \n",
       "37  05-2021                          Dominos     e commerce         180.0   \n",
       "\n",
       "                                                 type  \n",
       "0             [ Security questions, security answers]  \n",
       "1               [ Names, Aadhar number, bank details]  \n",
       "2   [comments, likes, reactions, account names, FB...  \n",
       "3   [names, email addresses, phone numbers, hashed...  \n",
       "4          [owner’s listed name, username, birthdate]  \n",
       "5   [phone numbers, home, and email addresses, int...  \n",
       "6                                          [password]  \n",
       "7                           [email address, password]  \n",
       "8   [voter Personal Identifiable Information (PII)...  \n",
       "9   [email addresses, usernames, DBKDF2 password h...  \n",
       "10  [internal ID, username, email, encrypted passw...  \n",
       "11  [email addresses, IP addresses, usernames, pas...  \n",
       "12  [names, home addresses, phone numbers, dates o...  \n",
       "13  [names, e-mail addresses, physical addresses, ...  \n",
       "14  [email addresses, names, usernames, cities, pa...  \n",
       "15  [names, email addresses, place of employment, ...  \n",
       "16    [names, birthdates, passwords, email addresses]  \n",
       "17  [email addresses, phone numbers, names, physic...  \n",
       "18      [names, email addresses, encrypted passwords]  \n",
       "19                              [usernames, password]  \n",
       "20                              [usernames, password]  \n",
       "21                       [emails, username,passwords]  \n",
       "22                       [email addresses, passwords]  \n",
       "23                       [email addresses, passwords]  \n",
       "24  [names, email addresses, mobile phone numbers,...  \n",
       "25  [shipping address records, account information...  \n",
       "26    [Usernames, Email addresses, Country, password]  \n",
       "27                                [login credentials]  \n",
       "28  [Account passwords, Account owner name, DOB, E...  \n",
       "29  [Name, Age, DOB, Gender, Location of message s...  \n",
       "30                           [debit card number, PIN]  \n",
       "31  [mobile numbers, partial account numbers, acco...  \n",
       "32  [names, mobile numbers, email ids, occupations...  \n",
       "33          [names, emails, phone numbers, addresses]  \n",
       "34  [email id, phone numbers, order details, addre...  \n",
       "35  [IDs, passwords, date joined, last login date,...  \n",
       "36  [passport, credit card details, birth dates, n...  \n",
       "37  [order details, email addresses, phone numbers...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7922b9de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x co-ordinate is time\n",
    "\n",
    "data2[\"datetime\"] = pd.to_datetime(data2[\"Date\"])\n",
    "x = [ int(i.month*((100)/(13)) + i.year*100) for i in data2[\"datetime\"] ]\n",
    "dates = list(data2[\"datetime\"])\n",
    "# data2[\"x\"] = x\n",
    "\n",
    "#  r represents \"size of breach\"; it's normalized to fir in a fixed range\n",
    "MAX_R, MIN_R = 30, 5\n",
    "r_ini = []\n",
    "for num, i in enumerate(data2[\"breach_count\"]):\n",
    "    r_ini.append( i*len((data2[\"type\"][num].split(\",\"))) )\n",
    "max_cal, min_cal = max(r_ini), min(r_ini)\n",
    "r = []\n",
    "for i in r_ini:\n",
    "    r.append(  (((i)/(max_cal-min_cal))*(MAX_R - MIN_R)) + MIN_R  )\n",
    "# data2['r'] = r"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75002b21",
   "metadata": {},
   "source": [
    "## data from online thing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6c74ad28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "539 5194604\n"
     ]
    }
   ],
   "source": [
    "temp_idx, temp = 0, 0\n",
    "for num, i in enumerate(data):\n",
    "    if i['PwnCount']*len(i[\"DataClasses\"]) > 0:\n",
    "        temp = i['PwnCount']*len(i[\"DataClasses\"])\n",
    "        temp_idx = num\n",
    "print(temp_idx, temp)\n",
    "del data[temp_idx]\n",
    "\n",
    "for i in data:\n",
    "    temp = pd.to_datetime(i[\"BreachDate\"])\n",
    "    x.append( int(temp.month*((100)/(13)) + temp.year*100) )\n",
    "    dates.append(temp)\n",
    "    r_ini.append( i['PwnCount']*len(i[\"DataClasses\"]) )\n",
    "\n",
    "r_ini = np.log(r_ini)\n",
    "max_cal, min_cal = max(r_ini), min(r_ini)\n",
    "r = []\n",
    "for i in r_ini:\n",
    "    r.append(  int((((i - min_cal)/(max_cal-min_cal))*(MAX_R - MIN_R)) + MIN_R)  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d0fd0def",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<ufunc 'log'>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b66ed65",
   "metadata": {},
   "source": [
    "## Making final JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b980a84c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# y represents company size\n",
    "\n",
    "MAX_SIZE = 100\n",
    "y = randint(MAX_SIZE, size=len(x), dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4108dd47",
   "metadata": {},
   "outputs": [],
   "source": [
    "#  categories for various datasets\n",
    "\n",
    "cat = list(data2[\"Category\"])\n",
    "cat += [\"NaN\"]*len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0445494c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_of_cats = {}\n",
    "for num, i in enumerate(cat):\n",
    "    if i in dict_of_cats:\n",
    "        dict_of_cats[i].append({\n",
    "                                \"x\": int(x[num]),\n",
    "                                \"y\": int(y[num]),\n",
    "                                \"r\": int(r[num])            \n",
    "                                })\n",
    "    else:\n",
    "        dict_of_cats[i] = [ {\n",
    "                                \"x\": int(x[num]),\n",
    "                                \"y\": int(y[num]),\n",
    "                                \"r\": int(r[num])            \n",
    "                                } ]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "626ebeaa",
   "metadata": {},
   "source": [
    "[\n",
    "      {\n",
    "        label: 'John',\n",
    "        data: [\n",
    "          {\n",
    "            x: 3,\n",
    "            y: 7,\n",
    "            r: 10\n",
    "          }\n",
    "        ],\n",
    "        backgroundColor:\"#ff6384\",\n",
    "        hoverBackgroundColor: \"#ff6384\"\n",
    "      },\n",
    "      {\n",
    "        label: 'George',\n",
    "          data: [\n",
    "            {\n",
    "              x: 1,\n",
    "              y: 2,\n",
    "              r: 10\n",
    "            }\n",
    "          ],\n",
    "          backgroundColor:\"#ff6384\",\n",
    "          hoverBackgroundColor: \"#ff6384\"\n",
    "      }\n",
    "      ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "15afebc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "final = {}\n",
    "final[\"dataset1\"] = [{\n",
    "                        \"label\":i,\n",
    "                        \"data\": dict_of_cats[i],\n",
    "                        \"backgroundColor\":\"#80008069\",\n",
    "                        \"hoverBackgroundColor\": \"#000000AA\"                    \n",
    "                    } for i in dict_of_cats ]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1814fb3",
   "metadata": {},
   "source": [
    "## Group by year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f9b77e16",
   "metadata": {},
   "outputs": [],
   "source": [
    "Df_group = pd.DataFrame([dates, x,y, r]).T\n",
    "Df_group.columns = [\"dates\", \"x\", \"y\", \"r\" ]\n",
    "Df_group[\"year\"] = Df_group.dates.dt.year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7a351dc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "Df_group[\"x\"] = Df_group[\"x\"].astype(\"str\").astype(\"int\")\n",
    "Df_group[\"y\"] = Df_group[\"y\"].astype(\"str\").astype(\"int\")\n",
    "Df_group[\"r\"] = Df_group[\"r\"].astype(\"str\").astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ca80cccc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishabh_/.local/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "year\n",
       "2007     1\n",
       "2008     3\n",
       "2009     2\n",
       "2010     3\n",
       "2011    17\n",
       "2012    19\n",
       "2013    33\n",
       "2014    49\n",
       "2015    75\n",
       "2016    94\n",
       "2017    54\n",
       "2018    77\n",
       "2019    60\n",
       "2020    68\n",
       "2021    22\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Df_group.groupby(\"year\")[\"x\", \"y\", \"r\"].agg([\"mean\", \"count\"])[\"x\"][\"count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1a900fe6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishabh_/.local/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "Df_group = Df_group.groupby(\"year\")[\"x\", \"y\", \"r\"].agg([\"mean\", \"count\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "22f1c733",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "year\n",
       "2007    200753.000000\n",
       "2008    200853.000000\n",
       "2009    200907.000000\n",
       "2010    201068.666667\n",
       "2011    201156.176471\n",
       "2012    201239.157895\n",
       "2013    201354.303030\n",
       "2014    201438.306122\n",
       "2015    201557.586667\n",
       "2016    201649.574468\n",
       "2017    201745.555556\n",
       "2018    201852.129870\n",
       "2019    201946.266667\n",
       "2020    202043.426471\n",
       "2021    202117.772727\n",
       "Name: mean, dtype: float64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for num, i in enumerate(Df_group[\"x\"][\"mean\"]):\n",
    "#     print(num, i)\n",
    "#     print(num, int(Df_group[\"y\"][\"mean\"]))\n",
    "#     print(num, int(Df_group[\"r\"][\"mean\"]))\n",
    "    \n",
    "Df_group[\"x\"][\"mean\"]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "66b514fc",
   "metadata": {},
   "source": [
    "final[\"dataset2\"] = [{\n",
    "                        \"label\": (2007+num),\n",
    "                        \"data\": [{\n",
    "                                    \"x\": int(i),\n",
    "                                    \"y\":int(Df_group[\"y\"][\"mean\"][num]),\n",
    "                                    \"r\": int(Df_group[\"r\"][\"mean\"][num])\n",
    "                                }],\n",
    "                        \"backgroundColor\":\"#80008069\",\n",
    "                        \"hoverBackgroundColor\": \"#000000AA\"\n",
    "                    }  for num,i in enumerate(Df_group[\"x\"][\"mean\"]) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cb3ac9e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "final[\"dataset2\"] = [{\n",
    "                        \"label\": (2007+num),\n",
    "                        \"data\":[{\n",
    "#                                     \"x\": int(i),\n",
    "                                     \"x\": (2007+num),\n",
    "                                    \"y\": 0,\n",
    "                                    \"r\": 0\n",
    "                                }],\n",
    "                        \"backgroundColor\":\"#80008069\",\n",
    "                        \"hoverBackgroundColor\": \"#000000AA\"\n",
    "                    } for num, i in enumerate(Df_group[\"x\"][\"mean\"])]\n",
    "\n",
    "for num, i in enumerate(Df_group[\"x\"][\"count\"]):\n",
    "    final[\"dataset2\"][num][\"data\"][0][\"y\"] = int(i)\n",
    "for num, i in enumerate(Df_group[\"r\"][\"mean\"]):\n",
    "    final[\"dataset2\"][num][\"data\"][0][\"r\"] = int(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3b35730d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 2007,\n",
       "  'data': [{'x': 2007, 'y': 1, 'r': 23}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2008,\n",
       "  'data': [{'x': 2008, 'y': 3, 'r': 23}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2009,\n",
       "  'data': [{'x': 2009, 'y': 2, 'r': 22}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2010,\n",
       "  'data': [{'x': 2010, 'y': 3, 'r': 22}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2011,\n",
       "  'data': [{'x': 2011, 'y': 17, 'r': 22}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2012,\n",
       "  'data': [{'x': 2012, 'y': 19, 'r': 22}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2013,\n",
       "  'data': [{'x': 2013, 'y': 33, 'r': 20}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2014,\n",
       "  'data': [{'x': 2014, 'y': 49, 'r': 20}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2015,\n",
       "  'data': [{'x': 2015, 'y': 75, 'r': 21}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2016,\n",
       "  'data': [{'x': 2016, 'y': 94, 'r': 21}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2017,\n",
       "  'data': [{'x': 2017, 'y': 54, 'r': 21}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2018,\n",
       "  'data': [{'x': 2018, 'y': 77, 'r': 21}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2019,\n",
       "  'data': [{'x': 2019, 'y': 60, 'r': 21}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2020,\n",
       "  'data': [{'x': 2020, 'y': 68, 'r': 21}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'},\n",
       " {'label': 2021,\n",
       "  'data': [{'x': 2021, 'y': 22, 'r': 19}],\n",
       "  'backgroundColor': '#80008069',\n",
       "  'hoverBackgroundColor': '#000000AA'}]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final[\"dataset2\"]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b8577bf4",
   "metadata": {},
   "source": [
    "out_file = open(\"./plotting/data/plotting_file2.json\", \"w\") \n",
    "    \n",
    "json.dump(final, out_file, indent = 4) \n",
    "    \n",
    "out_file.close() "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e78824cd",
   "metadata": {},
   "source": [
    "## co-occurence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4d40c37e",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_types = {}\n",
    "for i in data:\n",
    "    for j in i[\"DataClasses\"]:\n",
    "        if j not in unique_types:\n",
    "            unique_types[j] = 1\n",
    "        else:\n",
    "            unique_types[j] += 1\n",
    "DF_temp = pd.DataFrame(unique_types.keys(), unique_types.values()).reset_index()\n",
    "DF_temp.columns = [\"count\", \"Class\"]\n",
    "# DF_temp.to_csv(\"./classes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "739b7304",
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix = {i:0 for i in unique_types}\n",
    "Df_matrix = pd.DataFrame([matrix.values() for i in matrix], matrix.keys())\n",
    "Df_matrix.columns = matrix.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "5bffc12d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in Df_matrix:\n",
    "    for j in Df_matrix.index:\n",
    "        for point in data:\n",
    "            if ((str(i) in point[\"DataClasses\"]) and (str(j) in point[\"DataClasses\"])):\n",
    "                if str(i) == str(j):\n",
    "                    Df_matrix[i][j] += 2\n",
    "                else:\n",
    "                    Df_matrix[i][j] += 1\n",
    "                    Df_matrix[j][i] += 1\n",
    "for i in Df_matrix:\n",
    "    for j in Df_matrix.index:\n",
    "        Df_matrix[i][j] /= 2"
   ]
  },
  {
   "cell_type": "raw",
   "id": "f0333932",
   "metadata": {},
   "source": [
    "Df_matrix.to_csv(\"co-occurence_matrtix.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b7b0c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}