commitz.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, re, string, os, glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import wordpunct_tokenize, RegexpTokenizer\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import VotingClassifier, RandomForestClassifier\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import cross_val_score, cross_validate\n",
    "# import matplotlib.pyplot as plt\n",
    "from scipy.sparse import hstack, vstack, csr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_files = []\n",
    "# projects = ['abinit', 'libmesh', 'lammps', 'mdanalysis']\n",
    "projects = ['mdanalysis']\n",
    "for sp in projects:\n",
    "    s = \"/Users/saurabh/workspace/fss/project/data/data-collection/labeled_commits/human/{}/\".format(sp)\n",
    "    path = s\n",
    "    files = glob.glob(os.path.join(path, \"*.csv\"))\n",
    "    all_files.extend(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(all_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fname = \"/Users/saurabh/workspace/fss/project/data/data-collection/labeled_commits/auto/abinit_concat.csv\"\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [],
   "source": [
    "# raw_df = pd.read_csv(fname, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_from_each_file = (pd.read_csv(f) for f in all_files)\n",
    "concatenated_df   = pd.concat(df_from_each_file, ignore_index=True)\n",
    "raw_df = concatenated_df.drop_duplicates().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "buggy\n",
      "0.0    3761\n",
      "1.0     514\n",
      "Name: hash, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "buggies = raw_df.groupby(\"buggy\")[\"hash\"].count()\n",
    "print(buggies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove_n = 1000\n",
    "# drop_indices = np.random.choice(raw_df.index, remove_n, replace=False)\n",
    "# new_df = raw_df.drop(drop_indices)\n",
    "# b = new_df.groupby(\"buggy\")[\"hash\"].count()\n",
    "# print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index        int64\n",
      "hash        object\n",
      "time        object\n",
      "message     object\n",
      "buggy      float64\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(raw_df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4275, 5)\n",
      "(4275, 6)\n"
     ]
    }
   ],
   "source": [
    "# inft = []\n",
    "# for el in y:\n",
    "#     if not np.isfinite(el):\n",
    "#         inft.append(el)\n",
    "# print(el)\n",
    "print(raw_df.shape)\n",
    "raw_df = raw_df.dropna().reset_index()\n",
    "print(raw_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "level_0      int64\n",
       "index        int64\n",
       "hash        object\n",
       "time        object\n",
       "message     object\n",
       "buggy      float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_raw = raw_df['buggy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "rt = RegexpTokenizer(r'[^\\W_]+|[^\\W_\\s]+')\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "stemmer = SnowballStemmer(language='english')\n",
    "stopset = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    [new, builder, module, helper, for, creating, ...\n",
      "1    [writer, do, not, modify, coordinate, in, plac...\n",
      "2    [changed, generic, exception, to, specific, on...\n",
      "3      [updated, pypi, summary, with, mdanalysistests]\n",
      "4                               [merge, branch, build]\n",
      "Name: tknz_msg, dtype: object\n"
     ]
    }
   ],
   "source": [
    "raw_df['tknz_msg'] = raw_df['message'].apply(rt.tokenize).apply(lambda tkns: [lemmatizer.lemmatize(w.lower()) for w in tkns])\n",
    "print(raw_df['tknz_msg'].head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    [new, builder, module, helper, creating, charm...\n",
      "1         [writer, modify, coordinate, place, anymore]\n",
      "2    [changed, generic, exception, specific, one, d...\n",
      "3            [updated, pypi, summary, mdanalysistests]\n",
      "4                               [merge, branch, build]\n",
      "Name: msg, dtype: object\n"
     ]
    }
   ],
   "source": [
    "raw_df['msg'] = raw_df['tknz_msg']\\\n",
    "    .apply(lambda tkns: \\\n",
    "           list(filter(\\\n",
    "                       lambda word: word not in stopset \\\n",
    "                       and word not in string.punctuation\\\n",
    "                       and re.match(r'[^\\W\\d]*$', word) \\\n",
    "                       and len(word) > 2\\\n",
    "                       , tkns)))\n",
    "print(raw_df['msg'].head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "# words = raw_df['msg'].apply(pd.Series).stack().drop_duplicates().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(words))\n",
    "# print(words[:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df['msg_str'] = raw_df['msg'].apply(lambda tkns: ' '.join(tkns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "# no_features = min(1000, len(words))\n",
    "no_features = 100\n",
    "\n",
    "# tfidf_vectorizer = TfidfVectorizer(max_features=no_features)\n",
    "tfidf_vectorizer = TfidfVectorizer(ngram_range=(1, 3), max_df=0.95, min_df=2, max_features=no_features)\n",
    "X_tfidf = tfidf_vectorizer.fit_transform(raw_df['msg_str'])\n",
    "tfidf_feature_names = tfidf_vectorizer.get_feature_names()\n",
    "\n",
    "tf_vectorizer = CountVectorizer(max_features=no_features)\n",
    "# tf_vectorizer = CountVectorizer(ngram_range=(1, 3), max_df=0.95, min_df=2, max_features=no_features)\n",
    "X_tf = tf_vectorizer.fit_transform(raw_df['msg_str'])\n",
    "tf_feature_names = tf_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf-idf vectorized:  (4275, 100)\n",
      "tf vectorized:  (4275, 100)\n"
     ]
    }
   ],
   "source": [
    "print(\"tf-idf vectorized: \", X_tfidf.shape)\n",
    "print(\"tf vectorized: \", X_tf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tf_dense = X_tf.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "version bumped\n",
      "[('bumped', 1), ('version', 1)]\n"
     ]
    }
   ],
   "source": [
    "num = 2786\n",
    "print(raw_df.loc[num]['msg_str'])\n",
    "val = []\n",
    "for idx, d in enumerate(X_tf[num][0].toarray()[0]):\n",
    "#     print(d)\n",
    "    if d > 0:\n",
    "        val.append((tf_feature_names[idx], d))\n",
    "print(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_topics = 2\n",
    "num_iter = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nmf = NMF(n_components=no_topics, random_state=1, alpha=.1, l1_ratio=.5, init='nndsvd').fit(X_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation(n_components=no_topics, max_iter=num_iter, learning_method='online', learning_offset=50.,random_state=9, evaluate_every=100).fit(X_tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4275, 2)\n"
     ]
    }
   ],
   "source": [
    "lda_x = lda.transform(X_tf)\n",
    "print(lda_x.shape)\n",
    "# print(type(lda_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "key_words = [\"bug\", \"fix\", \"wrong\", \"error\", \"fail\", \"problem\", \"patch\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_topics(model, feature_names, no_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "#         print(\"Topic %d:\" % (topic_idx))\n",
    "        top_features = []\n",
    "        if topic_idx not in topic_word_prob:\n",
    "            topic_word_prob[topic_idx] = []\n",
    "        top_features = [(feature_names[i], topic[i]) for i in topic.argsort()[:-no_top_words - 1:-1]]\n",
    "        topic_word_prob[topic_idx] = top_features\n",
    "#         print(\" \".join(top_features))\n",
    "        feature_set = set([val[0] for val in top_features])\n",
    "        feature_names_set.update(feature_set)\n",
    "        print(\"Topic %d:\" % (topic_idx))\n",
    "        print(\", \".join([val[0] for val in top_features]))\n",
    "        word_list_for_topic = [val[0] for val in top_features]\n",
    "        word_probs_for_topic = [val[1] for val in top_features]\n",
    "        bug_prob_sum = 0\n",
    "        for bug_word in key_words:\n",
    "            if bug_word in word_list_for_topic:\n",
    "                bug_prob_sum += word_probs_for_topic[word_list_for_topic.index(bug_word)]\n",
    "        topic_indices.append(bug_prob_sum)\n",
    "#         if any(bug_word in [val[0] for val in top_features] for bug_word in key_words):\n",
    "#             topic_indices.append(topic_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0:\n",
      "doc, mdanalysis, merge, develop, code, branch, pull, request, updated, changelog\n",
      "Topic 1:\n",
      "test, added, fixed, fix, issue, file, analysis, new, atom, atomgroup\n",
      "[0, 470.65816498873426]\n",
      "1\n",
      "{0: [('doc', 576.716419528072), ('mdanalysis', 546.4453778732027), ('merge', 542.1957422323736), ('develop', 350.09954043510726), ('code', 276.595474398765), ('branch', 271.2377986311832), ('pull', 268.959295584659), ('request', 268.0170921389495), ('updated', 264.09941022535656), ('changelog', 187.14199077994402)], 1: [('test', 847.9450003111815), ('added', 686.35332450285), ('fixed', 526.1258470961051), ('fix', 470.65816498873426), ('issue', 392.03110375339156), ('file', 289.7254469102167), ('analysis', 260.58843106152295), ('new', 220.62921090588392), ('atom', 218.916674244189), ('atomgroup', 206.80073129988378)]}\n",
      "['test', 'merge', 'new', 'pull', 'changelog', 'develop', 'issue', 'branch', 'doc', 'fixed', 'atomgroup', 'added', 'atom', 'mdanalysis', 'updated', 'code', 'fix', 'file', 'request', 'analysis']\n"
     ]
    }
   ],
   "source": [
    "no_top_words = 10\n",
    "topic_indices = []\n",
    "topic_word_prob = {}\n",
    "feature_names_set = set()\n",
    "\n",
    "display_topics(lda, tf_feature_names, no_top_words)\n",
    "feature_names_list = list(feature_names_set)\n",
    "print(topic_indices)\n",
    "print(topic_indices.index(max(topic_indices)))\n",
    "print(topic_word_prob)\n",
    "print(feature_names_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_top_map_cache = {}\n",
    "def get_topic_top_words(model, feature_names):\n",
    "    if str(model) in model_top_map_cache:\n",
    "        return model_top_map_cache[str(model)]\n",
    "    topic_top_words = []\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "            top_words = [feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]\n",
    "            topic_top_words.append(top_words)\n",
    "    model_top_map_cache[str(model)] = topic_top_words\n",
    "    return topic_top_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_topics(words, no_top_words, model, feature_names):\n",
    "    topic_ranks = []\n",
    "    topic_top_words = get_topic_top_words(model, feature_names)\n",
    "    for top_words in topic_top_words:\n",
    "        topic_freq = 0\n",
    "        for w in words:\n",
    "            if w in top_words:\n",
    "                topic_freq += 1\n",
    "        topic_ranks.append(topic_freq)\n",
    "    buggy_topic = 0\n",
    "    max_val = max(topic_ranks)\n",
    "    idx = topic_ranks.index(max_val)\n",
    "    if idx in topic_indices:\n",
    "        buggy_topic = 1\n",
    "    return max_val, idx, buggy_topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df['topic_freq'], raw_df['topic_id'], raw_df['buggy_topic'] = zip(*raw_df['msg'].apply(lambda tkns: get_top_topics(tkns, 20, lda, tf_feature_names)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(raw_df.head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic_id  buggy\n",
      "0         0.0      2213\n",
      "          1.0        93\n",
      "1         0.0      1548\n",
      "          1.0       421\n",
      "dtype: int64\n",
      "index:  (0, 0.0) value:  2213\n",
      "index:  (0, 1.0) value:  93\n"
     ]
    }
   ],
   "source": [
    "tops_labels = raw_df.groupby(['topic_id','buggy']).size()\n",
    "print(tops_labels)\n",
    "for i, v in tops_labels.items():\n",
    "    if i[0] in topic_indices:\n",
    "        print('index: ', i, 'value: ', v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(lda_x[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4275, 14)\n"
     ]
    }
   ],
   "source": [
    "for i in range(no_topics):\n",
    "    topic_name = \"Topic_{}\".format(str(i))\n",
    "    raw_df[topic_name] = pd.Series(lda_x[:, i])\n",
    "print(raw_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "level_0          int64\n",
       "index            int64\n",
       "hash            object\n",
       "time            object\n",
       "message         object\n",
       "buggy          float64\n",
       "tknz_msg        object\n",
       "msg             object\n",
       "msg_str         object\n",
       "topic_freq       int64\n",
       "topic_id         int64\n",
       "buggy_topic      int64\n",
       "Topic_0        float64\n",
       "Topic_1        float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 360,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic0_word_prob_map = {}\n",
    "topic1_word_prob_map = {}\n",
    "topic_index1 = 1\n",
    "topic_index0 = 0\n",
    "for d in topic_word_prob[topic_index0]:\n",
    "        topic0_word_prob_map[d[0]] = d[1]\n",
    "for d in topic_word_prob[topic_index1]:\n",
    "        topic1_word_prob_map[d[0]] = d[1]\n",
    "def doc_word_mapping(words, topic0, topic1, feature_names):\n",
    "    is_max_0 = True\n",
    "    topic_prob0 = topic0\n",
    "    topic_prob1 = topic1\n",
    "    if topic_prob1 > topic_prob0:\n",
    "        is_max_0 = False\n",
    "    weighted_words = [0]*(len(feature_names_list)+3)\n",
    "    weighted_words[-2] = topic_prob1\n",
    "    weighted_words[-3] = topic_prob0\n",
    "    uniq_words = Counter(words)\n",
    "    for idx, w in enumerate(feature_names_list):\n",
    "        count = 0\n",
    "        if w in uniq_words:\n",
    "            count = uniq_words[w]\n",
    "        if w in key_words:\n",
    "            weighted_words[-1] = count * 1\n",
    "        prob = 0\n",
    "        if is_max_0 and  w in topic0_word_prob_map:\n",
    "            prob += count * topic0_word_prob_map[w]\n",
    "        if not is_max_0 and w in topic1_word_prob_map:\n",
    "            prob += count * topic1_word_prob_map[w]\n",
    "        weighted_words[idx] = prob\n",
    "    return weighted_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df['word_prob'] = raw_df.apply(lambda x: doc_word_mapping(x['msg'], x['Topic_0'], x['Topic_1'], tf_feature_names), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0, 0.0, 0, 0, 0, 0.0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0, 0.0, 0.0, 0, 0.0, 0.16707261563090586, 0.8329273843690941, 0]\n"
     ]
    }
   ],
   "source": [
    "print(raw_df['word_prob'][1278])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "def keyword_buggy(tokens):\n",
    "    return 1 if any(token in key_words for token in tokens) else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4275, 16)\n"
     ]
    }
   ],
   "source": [
    "raw_df['k_buggy'] = raw_df['msg'].apply(keyword_buggy)\n",
    "print(raw_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.DataFrame()\n",
    "final_df = raw_df[raw_df['buggy'] == 1]\n",
    "if len(topic_indices) > 100:\n",
    "    for idx in topic_indices:\n",
    "        final_df = final_df.append(raw_df[raw_df['topic_id'] == idx])\n",
    "    test_df = raw_df[(raw_df['hash'].apply(lambda x: x not in final_df['hash'].values))]\n",
    "else:\n",
    "    final_df = raw_df\n",
    "# final_df = raw_df\n",
    "use_all_as_test = False\n",
    "if not test_df.empty and test_df.size > 0:\n",
    "    use_all_as_test = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cm(y_true, y_pred):\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "\n",
    "    acc = (tp + tn) / (tn + fp + fn + tp)\n",
    "    prec = tp / (tp + fp)\n",
    "    rec = tp / (tp + fn)\n",
    "    f1 = 2 * prec * rec / (prec + rec)\n",
    "\n",
    "    beta = 3\n",
    "    f2 = (1+np.power(beta, 2))*prec*rec/(np.power(beta,2)*prec + rec)\n",
    "\n",
    "    print(\"accuracy\", acc)\n",
    "    print(\"precision\", prec)\n",
    "    print(\"recall\", rec)\n",
    "    print(\"f1\", f1)\n",
    "    print(\"f2\", f2)\n",
    "    print(tn, fp, fn, tp)\n",
    "    return acc, prec, rec, f1, f2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = raw_df['buggy']\n",
    "# y_lda = raw_df['topic_id']\n",
    "y_lda = raw_df['buggy_topic']\n",
    "y_kwd = raw_df['k_buggy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LDA:\n",
      "accuracy 0.383859649122807\n",
      "precision 0.040329575021682565\n",
      "recall 0.18093385214007782\n",
      "f1 0.06595744680851064\n",
      "f2 0.1341604154645124\n",
      "1548 2213 421 93\n",
      "\n",
      "Keywords:\n",
      "accuracy 0.8444444444444444\n",
      "precision 0.38184663536776214\n",
      "recall 0.47470817120622566\n",
      "f1 0.42324371205550737\n",
      "f2 0.46343779677113006\n",
      "3366 395 270 244\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8444444444444444,\n",
       " 0.38184663536776214,\n",
       " 0.47470817120622566,\n",
       " 0.42324371205550737,\n",
       " 0.46343779677113006)"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"LDA:\")\n",
    "cm(y_true, y_lda)\n",
    "print(\"\")\n",
    "print(\"Keywords:\")\n",
    "cm(y_true, y_kwd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4275, 9)\n",
      "(4275, 9)\n"
     ]
    }
   ],
   "source": [
    "print(raw_df.shape)\n",
    "print(final_df.shape)\n",
    "if use_all_as_test:\n",
    "    print(test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# buggies = final_df.groupby(\"buggy\")[\"hash\"].count()\n",
    "# print(buggies)\n",
    "# ratio = buggies[1]/buggies[0]\n",
    "# min_ratio = 0.1\n",
    "# if ratio < min_ratio:\n",
    "#     buggy_indices = set(final_df.index[final_df['buggy'] == 1].tolist())\n",
    "#     rem = buggies[0] - int(buggies[1]/min_ratio)\n",
    "#     drop_indices_rand = set(np.random.choice(final_df.index, rem, replace=False))\n",
    "#     drop_indices = drop_indices_rand - buggy_indices\n",
    "#     df_subset = final_df.drop(drop_indices)\n",
    "#     buggies_new = df_subset.groupby(\"buggy\")[\"hash\"].count()\n",
    "#     final_df = df_subset\n",
    "#     print(buggies_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_X_tf = tf_vectorizer.transform(final_df['msg_str'])\n",
    "# X = lda.transform(final_X_tf)\n",
    "# X = tfidf_vectorizer.transform(final_df['msg_str'])\n",
    "X = final_df['word_prob'].values.tolist()\n",
    "y = final_df['buggy']\n",
    "if use_all_as_test:\n",
    "#     test_X_tf = tf_vectorizer.transform(test_df['msg_str'])\n",
    "#     test_X = lda.transform(test_X_tf)\n",
    "#     test_X = tfidf_vectorizer.transform(test_df['msg_str'])\n",
    "    test_X = test_df['word_prob'].values.tolist()\n",
    "    test_y = test_df['buggy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(X[1])\n",
    "# print(y.shape)\n",
    "# # X = hstack((X,np.array(final_df['topic_id'])[:,None]))\n",
    "# # X = hstack((X,np.array(final_df['topic_freq'])[:,None]))\n",
    "# # X = hstack((X,np.array(final_df['buggy_topic'])[:,None]))\n",
    "# print(X.shape)\n",
    "# if use_all_as_test:\n",
    "# #     test_X = hstack((test_X,np.array(test_df['topic_id'])[:,None]))\n",
    "# #     test_X = hstack((test_X,np.array(test_df['topic_freq'])[:,None]))\n",
    "# #     test_X = hstack((test_X,np.array(test_df['buggy_topic'])[:,None]))\n",
    "#     print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=9, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# positive in train set: 411 \n",
      "# negative in train set: 3009\n",
      "# positive in test set: 103 \n",
      "# negative in test set: 752\n"
     ]
    }
   ],
   "source": [
    "print(\"# positive in train set: {}\".format(len(y_train[y_train == 1])),\n",
    "      \"\\n# negative in train set: {}\".format(len(y_train[y_train == 0])))\n",
    "print(\"# positive in test set: {}\".format(len(y_test[y_test == 1])),\n",
    "      \"\\n# negative in test set: {}\".format(len(y_test[y_test == 0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler().fit(X_train)\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4900,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=100, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='linear', max_iter=-1, probability=False, random_state=9,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clf = MultinomialNB()\n",
    "# clf = LinearSVC(C=100, loss='hinge', random_state=9, max_iter=500000)\n",
    "clf = SVC(C=100, kernel='linear', random_state=9)\n",
    "# clf = DecisionTreeClassifier(random_state=0)\n",
    "# clf = RandomForestClassifier(n_estimators=100, random_state=9)\n",
    "\n",
    "clf.fit(X_train, y_train)\n",
    "# y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([454, 391], dtype=int32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.n_support_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4903,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scoring = ['precision_macro', 'recall_macro']\n",
    "# estimators = []\n",
    "# clf1 = MultinomialNB()\n",
    "# estimators.append(clf1)\n",
    "# clf2 = SVC(C=100, kernel='linear')\n",
    "# estimators.append(clf2)\n",
    "# clf3 = SVC(C=100, kernel='rbf', gamma=0.01)\n",
    "# estimators.append(clf3)\n",
    "# eclf = VotingClassifier(estimators=[('nb', clf1), ('svml', clf2), ('svmr', clf3)], voting='hard')\n",
    "# for clf, label in zip([clf1, clf2, clf3, eclf], ['Naive Bayes', 'SVM Linear', 'SVM RBF', 'Ensemble']):\n",
    "#     scores = cross_val_score(clf, X, y, cv=5, scoring='f1')\n",
    "#     print(\"F1 : %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4904,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clf = eclf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4905,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if use_all_as_test:\n",
    "#     mod_X_test = vstack((test_X, X_test)).todense()\n",
    "#     print(mod_X_test.shape)\n",
    "#     pred_y = clf.predict(mod_X_test)\n",
    "# #     pred_y = clf.predict(X_test)\n",
    "# else:\n",
    "#     pred_y = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(855,)\n",
      "(855,)\n"
     ]
    }
   ],
   "source": [
    "p_pred_y = clf.predict(X_test)\n",
    "print(p_pred_y.shape)\n",
    "if use_all_as_test:\n",
    "    p_pred_y = np.append(p_pred_y, clf.predict(test_X))\n",
    "print(p_pred_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()\n",
    "if use_all_as_test:\n",
    "    mod_y_test = np.append(test_y, y_test)\n",
    "    print(mod_y_test.shape)\n",
    "#     mod_y_test = y_test\n",
    "else:\n",
    "    mod_y_test = y_test\n",
    "tn, fp, fn, tp = confusion_matrix(mod_y_test, p_pred_y).ravel()\n",
    "\n",
    "acc = (tp + tn) / (tn + fp + fn + tp)\n",
    "prec = tp / (tp + fp)\n",
    "rec = tp / (tp + fn)\n",
    "f1 = 2 * prec * rec / (prec + rec)\n",
    "\n",
    "beta = 3\n",
    "f2 = (1+np.power(beta, 2))*prec*rec/(np.power(beta,2)*prec + rec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.8760233918128655\n",
      "precision 0.47692307692307695\n",
      "recall 0.30097087378640774\n",
      "f1 0.36904761904761907\n",
      "f2 0.31249999999999994\n",
      "718 34 72 31\n",
      "['mdanalysis']\n"
     ]
    }
   ],
   "source": [
    "print(\"accuracy\", acc)\n",
    "print(\"precision\", prec)\n",
    "print(\"recall\", rec)\n",
    "print(\"f1\", f1)\n",
    "print(\"f2\", f2)\n",
    "print(tn, fp, fn, tp)\n",
    "print(projects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4909,
   "metadata": {},
   "outputs": [],
   "source": [
    "# libmesh_fastread_fname = \"/Users/saurabh/workspace/fss/project/data/data-collection/labeled_commits/fastread/abinit_fast_labeled.csv\"\n",
    "# libmesh_df = pd.read_csv(libmesh_fastread_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4910,
   "metadata": {},
   "outputs": [],
   "source": [
    "# libmesh_df.columns = [\"hash\", \"abstract\", 'year', \"lnk\", \"label\", \"code\", \"time\"]\n",
    "# codes = libmesh_df.groupby(\"code\")[\"hash\"].count()\n",
    "# print(codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4911,
   "metadata": {},
   "outputs": [],
   "source": [
    "# raw_df['hash'] = raw_df['hash'].astype(str)\n",
    "# libmesh_df['hash'] = libmesh_df['hash'].astype(str)\n",
    "# print(libmesh_df.shape)\n",
    "# print(raw_df.shape)\n",
    "# print(libmesh_df.dtypes)\n",
    "# print(raw_df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4912,
   "metadata": {},
   "outputs": [],
   "source": [
    "# libmesh_df_merged = pd.merge(raw_df, libmesh_df, how='inner', on=['hash'], suffixes=(\"_raw\", \"_libmesh\"))\n",
    "# print(libmesh_df_merged[['hash', 'msg_str', 'buggy_topic', 'buggy', 'code']].head(10))\n",
    "# print(libmesh_df_merged.shape)\n",
    "# libmesh_df_merged_d = libmesh_df_merged[['hash', 'msg_str', 'buggy_topic', 'buggy', 'code']]\n",
    "# print(libmesh_df_merged_d.head(10))\n",
    "# print(libmesh_df_merged_d.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4913,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_libmesh = tf_vectorizer.transform(libmesh_df_merged_d['msg_str'])\n",
    "# X_libmesh = hstack((X_libmesh,np.array(libmesh_df_merged_d['buggy_topic'])[:,None]))\n",
    "# print(X_libmesh.shape)\n",
    "# libmesh_df_merged_d['pred_code'] = clf.predict(X_libmesh)\n",
    "# print(libmesh_df_merged_d.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4914,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_df = libmesh_df_merged_d[['buggy', 'code', 'pred_code']]\n",
    "# print(y_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4915,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_df = y_df[y_df['code'] != 'undetermined']\n",
    "# print(y_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4916,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_knn = NearestNeighbors(metric='cosine', algorithm='brute')\n",
    "# model_knn.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4917,
   "metadata": {},
   "outputs": [],
   "source": [
    "# k = 10\n",
    "# km = KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=5, verbose=1)\n",
    "# km.fit(X)\n",
    "# %matplotlib inline\n",
    "\n",
    "# plt.hist(km.labels_, bins=k)\n",
    "# plt.show()\n",
    "# plt.close()\n",
    "\n",
    "# cluster_assignments_dict = {}\n",
    "# # print(np.where(km.labels_ == i))\n",
    "# # print(raw_df.iloc[79]['msg'])\n",
    "# for i in set(km.labels_):\n",
    "# #     print(i)\n",
    "#     current_cluster_vals = [(raw_df.iloc[x]['msg'], raw_df.iloc[x]['buggy']) for x in np.where(km.labels_ == i)[0]]\n",
    "#     cluster_assignments_dict[i] = current_cluster_vals\n",
    "\n",
    "# cluster_pick = np.random.choice(len(set(km.labels_)))\n",
    "# print('Cluster {0}'.format(cluster_pick))\n",
    "# cluster_assignments_dict[cluster_pick]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}