Very Last Sim Feature.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.\n",
      "  \"`IPython.html.widgets` has moved to `ipywidgets`.\", ShimWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import sparse\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from tqdm import tqdm\n",
    "import igraph\n",
    "from sklearn.neighbors import LSHForest\n",
    "from sklearn.preprocessing import normalize\n",
    "import random\n",
    "from sklearn.cluster import SpectralClustering\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.metrics import euclidean_distances\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "import math\n",
    "from sklearn.metrics.pairwise import manhattan_distances\n",
    "from scipy.stats import entropy\n",
    "import os\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "\n",
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_latent = 100 # number of latent variables in LDA\n",
    "n_components = 200 # number of components in SVD\n",
    "rare_thresh = 1 # if song occurs less than rare_thresh times in train set we replace its score by its genre score\n",
    "pair_type = 'genre' # media or genre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_bow_matrix(data, pair_type):\n",
    "# return matrix of size n_users*n_medias with cf_mat[i,j]==n iff user i listened to media j n times\n",
    "# and mappings user_id -> row of matrix, media_id -> column of matrix\n",
    "    positive_data = data.loc[data['is_listened']==1]\n",
    "    user_ids = dict([(j,i) for  i,j in enumerate(positive_data['user_id'].unique())])\n",
    "    media_ids = dict([(j,i) for  i,j in enumerate(positive_data[pair_type+'_id'].unique())])\n",
    "    n_users = len(user_ids)\n",
    "    n_medias = len(media_ids)\n",
    "    bow = sparse.dok_matrix((n_users,n_medias))\n",
    "    for r in tqdm(positive_data.iterrows()):\n",
    "        if r[1]['is_listened']:\n",
    "            bow[user_ids[r[1]['user_id']],media_ids[r[1][pair_type+'_id']]] += 1\n",
    "    return user_ids, media_ids, bow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_tfidf_matrix(bow_matrix):\n",
    "# return reweighted bow matrix\n",
    "    # how many users have listened to each song\n",
    "    media_occurences = np.log(1.+np.array((bow_matrix>0).sum(axis=0))).squeeze()\n",
    "    # to how many songs has listened each user\n",
    "    user_occurences = np.array((bow_matrix).sum(axis=1))\n",
    "    tfidf_matrix = sparse.dok_matrix(bow_matrix.shape)\n",
    "    for idx, value in tqdm(bow_matrix.items()):\n",
    "        tfidf_matrix[idx[0],idx[1]] = value / user_occurences[idx[0]]\n",
    "        tfidf_matrix[idx[0],idx[1]] /= media_occurences[idx[1]]\n",
    "    return tfidf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add_ids(ids, vectors):\n",
    "# add ids from dictionary 'ids' to matrix 'vectors' as a column\n",
    "    vectors_ext = np.zeros([vectors.shape[0], vectors.shape[1]+1])\n",
    "    vectors_ext[:vectors.shape[0],:vectors.shape[1]] = vectors\n",
    "    for j,i in ids.iteritems():\n",
    "        vectors_ext[i,vectors.shape[1]] = j\n",
    "    return vectors_ext\n",
    "\n",
    "def extract_ids(vectors_ext):\n",
    "# extract ids to dictionary 'ids' from last column of matrix 'vectors_ext'\n",
    "    vectors = vectors_ext[:vectors_ext.shape[0],:-1]\n",
    "    ids = {}\n",
    "    for i in range(vectors_ext.shape[0]):\n",
    "        ids[vectors_ext[i,-1]] = i\n",
    "    return ids, vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def histogram_intersection(v1,v2):\n",
    "    return np.sum(np.min(np.array([v1.squeeze(),v2.squeeze()]), axis=0))\n",
    "def hellinger_distance(v1,v2):\n",
    "    return euclidean_distances(np.sqrt(v1),np.sqrt(v2))/math.sqrt(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = os.path.join(os.curdir,'new_user_'+pair_type+'_features')\n",
    "if 'new_user_'+pair_type+'_features' not in os.listdir(os.curdir):\n",
    "    os.mkdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('data/train.csv',sep=',', usecols=[pair_type+'_id','user_id','is_listened'])\n",
    "test_data = pd.read_csv('data/test.csv',sep=',', usecols=['sample_id',pair_type+'_id','user_id'], index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5170492it [17:58, 4795.80it/s]\n"
     ]
    }
   ],
   "source": [
    "user_ids, media_ids, bow_matrix = get_bow_matrix(train_data, pair_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 350605/350605 [01:01<00:00, 5746.99it/s]\n"
     ]
    }
   ],
   "source": [
    "tfidf_matrix = get_tfidf_matrix(bow_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LDA features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation(n_topics=n_latent, learning_method='batch').fit(bow_matrix)\n",
    "user_lda_vectors = lda.transform(bow_matrix)\n",
    "media_lda_vectors = lda.components_.T\n",
    "user_lda_vectors = normalize(user_lda_vectors, norm='l1')\n",
    "media_lda_vectors = normalize(media_lda_vectors, norm='l1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "user_lda_vectors_ext = add_ids(user_ids, user_lda_vectors)\n",
    "with open(os.path.join(path,'user_lda_vectors.npy'),'w') as f:\n",
    "    np.save(f, user_lda_vectors_ext)\n",
    "media_lda_vectors_ext = add_ids(media_ids, media_lda_vectors)\n",
    "with open(os.path.join(path,pair_type+'_lda_vectors.npy'),'w') as f:\n",
    "    np.save(f, media_lda_vectors_ext)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LSI features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "user_lsi_vectors,s,vt = sparse.linalg.svds(tfidf_matrix, k=n_components)\n",
    "media_lsi_vectors = vt.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "user_lsi_vectors_ext = add_ids(user_ids, user_lsi_vectors)\n",
    "with open(os.path.join(path,'user_lsi_vectors.npy'),'w') as f:\n",
    "    np.save(f, user_lsi_vectors_ext)\n",
    "media_lsi_vectors_ext = add_ids(media_ids, media_lsi_vectors)\n",
    "with open(os.path.join(path,pair_type+'_lsi_vectors.npy'),'w') as f:\n",
    "    np.save(f, media_lsi_vectors_ext)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LDA-based similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(path,'user_lda_vectors.npy'),'r') as f:\n",
    "    user_lda_vectors_ext = np.load(f)\n",
    "user_ids, user_lda_vectors = extract_ids(user_lda_vectors_ext)\n",
    "with open(os.path.join(path,pair_type+'_lda_vectors.npy'),'r') as f:\n",
    "    media_lda_vectors_ext = np.load(f)\n",
    "media_ids, media_lda_vectors = extract_ids(media_lda_vectors_ext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lda_counter = 0\n",
    "def lda_similarity(user_id,media_id):\n",
    "    # return Hellinger similarity between users and medias\n",
    "    # return 0 for unknown users and medias\n",
    "    global lda_counter\n",
    "    lda_counter += 1\n",
    "    if lda_counter%300000==0:\n",
    "        print lda_counter\n",
    "    if user_id in user_ids and media_id in media_ids:\n",
    "        i = user_ids[user_id]\n",
    "        j = media_ids[media_id]\n",
    "        return float(1-hellinger_distance(media_lda_vectors[j,:].reshape([1,-1]),user_lda_vectors[i,:].reshape([1,-1])))\n",
    "    return 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300000\n",
      "600000\n",
      "900000\n",
      "1200000\n",
      "1500000\n",
      "1800000\n",
      "2100000\n",
      "2400000\n",
      "2700000\n",
      "3000000\n",
      "3300000\n",
      "3600000\n",
      "3900000\n",
      "4200000\n",
      "4500000\n",
      "4800000\n",
      "5100000\n",
      "5400000\n",
      "5700000\n",
      "6000000\n",
      "6300000\n",
      "6600000\n",
      "6900000\n",
      "7200000\n",
      "7500000\n"
     ]
    }
   ],
   "source": [
    "train_data['lda_similarity'] = train_data[['user_id',pair_type+'_id']].apply(lambda rec : lda_similarity(rec['user_id'],rec[pair_type+'_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data['lda_similarity'] = test_data[['user_id',pair_type+'_id']].apply(lambda rec : lda_similarity(rec['user_id'],rec[pair_type+'_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data[['lda_similarity']].to_csv(os.path.join(path,'./train_lda_similarity.csv'))\n",
    "test_data[['lda_similarity']].to_csv(os.path.join(path,'./test_lda_similarity.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LSI-based similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(path,'user_lsi_vectors.npy'),'r') as f:\n",
    "    user_lsi_vectors_ext = np.load(f)\n",
    "user_ids, user_lsi_vectors = extract_ids(user_lsi_vectors_ext)\n",
    "with open(os.path.join(path,pair_type+'_lsi_vectors.npy'),'r') as f:\n",
    "    media_lsi_vectors_ext = np.load(f)\n",
    "media_ids, media_lsi_vectors = extract_ids(media_lsi_vectors_ext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsi_counter = 0\n",
    "def lsi_similarity(user_id,media_id):\n",
    "    # return cosine similarity between users and medias\n",
    "    # return 0 for unknown users and medias\n",
    "    global lsi_counter\n",
    "    lsi_counter += 1\n",
    "    if lsi_counter%300000==0:\n",
    "        print lsi_counter\n",
    "    if user_id in user_ids and media_id in media_ids:\n",
    "        i = user_ids[user_id]\n",
    "        j = media_ids[media_id]\n",
    "        return float(cosine_similarity(user_lsi_vectors[i,:].reshape([1,-1]),media_lsi_vectors[j,:].reshape([1,-1])))\n",
    "    return 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300000\n",
      "600000\n",
      "900000\n",
      "1200000\n",
      "1500000\n",
      "1800000\n",
      "2100000\n",
      "2400000\n",
      "2700000\n",
      "3000000\n",
      "3300000\n",
      "3600000\n",
      "3900000\n",
      "4200000\n",
      "4500000\n",
      "4800000\n",
      "5100000\n",
      "5400000\n",
      "5700000\n",
      "6000000\n",
      "6300000\n",
      "6600000\n",
      "6900000\n",
      "7200000\n",
      "7500000\n"
     ]
    }
   ],
   "source": [
    "train_data['lsi_similarity'] = train_data[['user_id',pair_type+'_id']].apply(lambda rec : lsi_similarity(rec['user_id'],rec[pair_type+'_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data['lsi_similarity'] = test_data[['user_id',pair_type+'_id']].apply(lambda rec : lsi_similarity(rec['user_id'],rec[pair_type+'_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data[['lsi_similarity']].to_csv(os.path.join(path,'./train_lsi_similarity.csv'))\n",
    "test_data[['lsi_similarity']].to_csv(os.path.join(path,'./test_lsi_similarity.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combining both similarity measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data['lsi_similarity'] = pd.read_csv(os.path.join(path,'./test_lsi_similarity.csv'), index_col=0)\n",
    "train_data['lsi_similarity'] = pd.read_csv(os.path.join(path,'./train_lsi_similarity.csv'), index_col=0)\n",
    "test_data['lda_similarity'] = pd.read_csv(os.path.join(path,'./test_lda_similarity.csv'), index_col=0)\n",
    "train_data['lda_similarity'] = pd.read_csv(os.path.join(path,'./train_lda_similarity.csv'), index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# Normalizing scores\n",
    "lsi_scaler = StandardScaler().fit(train_data['lsi_similarity'].values)\n",
    "train_data['lsi_similarity'] = lsi_scaler.transform(train_data['lsi_similarity'].values)\n",
    "test_data['lsi_similarity'] = lsi_scaler.transform(test_data['lsi_similarity'].values)\n",
    "lda_scaler = StandardScaler().fit(train_data['lda_similarity'].values)\n",
    "train_data['lda_similarity'] = lda_scaler.transform(train_data['lda_similarity'].values)\n",
    "test_data['lda_similarity'] = lda_scaler.transform(test_data['lda_similarity'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = train_data[['lda_similarity','lsi_similarity']].values\n",
    "X_test = test_data[['lda_similarity','lsi_similarity']].values\n",
    "y_train = train_data['is_listened'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = LogisticRegression().fit(X_train,y_train)\n",
    "y_train_pred = clf.predict_proba(X_train)\n",
    "y_test_pred = clf.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data['mixed_similarity'] = y_train_pred[:,1]\n",
    "test_data['mixed_similarity'] = y_test_pred[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data[['mixed_similarity']].to_csv(os.path.join(path,'./train_mixed_similarity.csv'))\n",
    "test_data[['mixed_similarity']].to_csv(os.path.join(path,'./test_mixed_similarity.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replacing user-media similarities of rare songs by user-genre similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('data/train.csv',sep=',', usecols=['media_id','genre_id','user_id','is_listened'])\n",
    "test_data = pd.read_csv('data/test.csv',sep=',', usecols=['sample_id','media_id','genre_id','user_id'], index_col=0)\n",
    "train_data['genre_similarity'] = pd.read_csv('new_user_genre_features/train_mixed_similarity.csv', index_col=0)\n",
    "test_data['genre_similarity'] = pd.read_csv('new_user_genre_features/test_mixed_similarity.csv', index_col=0)\n",
    "train_data['media_similarity'] = pd.read_csv('new_user_media_features/train_mixed_similarity.csv', index_col=0)\n",
    "test_data['media_similarity'] = pd.read_csv('new_user_media_features/test_mixed_similarity.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# how many times each song was listened\n",
    "media_occurences = train_data[['media_id','is_listened']].groupby('media_id').count().reset_index().rename(columns={'is_listened':'media_occurences'})\n",
    "# how many times each genre was listened\n",
    "genre_occurences = train_data[['genre_id','is_listened']].groupby('genre_id').count().reset_index().rename(columns={'is_listened':'genre_occurences'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = pd.merge(test_data, media_occurences, on='media_id', how='left')\n",
    "test_data = pd.merge(test_data, genre_occurences, on='genre_id', how='left')\n",
    "test_data = test_data.fillna(0)\n",
    "test_data['mixed_similarity'] = test_data['media_similarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data.ix[test_data['media_occurences']<rare_thresh, 'mixed_similarity'] = test_data.ix[test_data['media_occurences']<rare_thresh, 'genre_similarity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data[['mixed_similarity']].to_csv(os.path.join('./test_mixed_media_genre_similarity.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = pd.merge(train_data, media_occurences, on='media_id', how='left')\n",
    "train_data = pd.merge(train_data, genre_occurences, on='genre_id', how='left')\n",
    "train_data = test_data.fillna(0)\n",
    "train_data['mixed_similarity'] = train_data['media_similarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data.ix[train_data['media_occurences']<rare_thresh, 'mixed_similarity'] = train_data.ix[train_data['media_occurences']<rare_thresh, 'genre_similarity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data[['mixed_similarity']].to_csv(os.path.join('./train_mixed_media_genre_similarity.csv'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}