prepare-svd-for-cite.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Prepare SVD for CITE**\n\nIn this Jupyter notebook, data from train and test datasets is put together and then the TruncatedSVD is calculated.\nAlso join data from a fix to test data the organizers have released in the middle of the competition.\n\nIn kaggle environment it is more convenient to do this in a separate notebook, as it would be a waste of both time and GPU quota to calculate the TruncatedSVD each time before fitting the model.","metadata":{}},{"cell_type":"code","source":"# Importing the libraries\n\nimport numpy as np\nimport pandas as pd\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport gc, pickle, scipy.sparse\nfrom more_itertools import sliced\nfrom sklearn.decomposition import TruncatedSVD\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-27T08:47:01.988619Z","iopub.execute_input":"2022-10-27T08:47:01.990459Z","iopub.status.idle":"2022-10-27T08:47:03.064794Z","shell.execute_reply.started":"2022-10-27T08:47:01.990319Z","shell.execute_reply":"2022-10-27T08:47:03.062951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Need this library to read *.h5 files\n!pip install --quiet tables","metadata":{"execution":{"iopub.status.busy":"2022-10-27T08:47:05.603668Z","iopub.execute_input":"2022-10-27T08:47:05.604323Z","iopub.status.idle":"2022-10-27T08:47:21.534436Z","shell.execute_reply.started":"2022-10-27T08:47:05.604273Z","shell.execute_reply":"2022-10-27T08:47:21.533014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/open-problems-multimodal/\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\nFP_CITE_TEST_INPUTS_FIX = os.path.join(DATA_DIR,\"test_cite_inputs_day_2_donor_27678.h5\")\n\nFP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\nFP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\nFP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-10-27T08:47:24.685994Z","iopub.execute_input":"2022-10-27T08:47:24.686512Z","iopub.status.idle":"2022-10-27T08:47:24.694953Z","shell.execute_reply.started":"2022-10-27T08:47:24.686465Z","shell.execute_reply":"2022-10-27T08:47:24.693744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the train dataset to a DataFrame\ndf = pd.read_hdf(FP_CITE_TRAIN_INPUTS)\nprint('import finished')","metadata":{"execution":{"iopub.status.busy":"2022-10-27T08:47:32.684191Z","iopub.execute_input":"2022-10-27T08:47:32.684665Z","iopub.status.idle":"2022-10-27T08:48:28.178700Z","shell.execute_reply.started":"2022-10-27T08:47:32.684632Z","shell.execute_reply":"2022-10-27T08:48:28.177301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Remove constant columns and chunk by chunk create sparse matrix from the DataFrame.\n# Then the DataFrame is deleted to free the RAM. \n\nconstant_cols = df.columns[df.nunique() <= 1]\nall_df_cols = df.columns\n\ndf = df.drop(columns=constant_cols)\nprint(f\"Original df shape: {str(df.shape):14} {df.size*4/1024/1024/1024:2.3f} GByte\")\ngc.collect()\n\nn = 2000  #chunk row size\nindex_slices = sliced(range(len(df)), n)\ni = 0\nfor index_slice in index_slices:\n    sparse_chunk = scipy.sparse.csr_matrix(df.iloc[index_slice].values)\n    if i == 0:\n        sparse_X = sparse_chunk\n    else:\n        sparse_X = scipy.sparse.vstack([sparse_X, sparse_chunk])\n    i = i + 1\n\ndel df, sparse_chunk, i\ngc.collect()\n\nprint(f\"sparse df shape: {str(sparse_X.shape):14} {sparse_X.size*4/1024/1024/1024:2.3f} GByte\")","metadata":{"execution":{"iopub.status.busy":"2022-10-22T11:40:30.218941Z","iopub.execute_input":"2022-10-22T11:40:30.219390Z","iopub.status.idle":"2022-10-22T11:41:39.722872Z","shell.execute_reply.started":"2022-10-22T11:40:30.219348Z","shell.execute_reply":"2022-10-22T11:41:39.721669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load test data and convert it to sparse matrix.\n# Do not read first 7476 rows as this data should be replaced by the data fix.\ndf_test = pd.read_hdf(FP_CITE_TEST_INPUTS, start=7476).drop(columns=constant_cols)\nprint(f\"Original Xt shape: {str(df_test.shape):14} {df_test.size*4/1024/1024/1024:2.3f} GByte\")\ngc.collect()\nindex_slices = sliced(range(len(df_test)), n)\nfor index_slice in index_slices:\n    sparse_chunk = scipy.sparse.csr_matrix(df_test.iloc[index_slice].values)\n    sparse_X = scipy.sparse.vstack([sparse_X, sparse_chunk])\n\ndel df_test, sparse_chunk\ngc.collect()\nprint(f\"Total sparse shape: {str(sparse_X.shape):14} {sparse_X.size*4/1024/1024/1024:2.3f} GByte\")\n","metadata":{"execution":{"iopub.status.busy":"2022-10-22T11:41:47.175720Z","iopub.execute_input":"2022-10-22T11:41:47.176493Z","iopub.status.idle":"2022-10-22T11:43:15.884573Z","shell.execute_reply.started":"2022-10-22T11:41:47.176449Z","shell.execute_reply":"2022-10-22T11:43:15.883225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same operation with test data fix. It is relatively small, so do not need chunks here.\n# For some reason, the fix data contains some additional columns not present neither in train nor test datasets.\n# So, I also delete those additional columns.\n\ndf_test_fix = pd.read_hdf(FP_CITE_TEST_INPUTS_FIX)\nfor col in df_test_fix.columns:\n    if col not in all_df_cols:\n        constant_cols = np.append(constant_cols, col)\ndf_test_fix = df_test_fix.drop(columns=constant_cols)\n\nprint(f\"Original Xt_fix shape: {str(df_test_fix.shape):14} {df_test_fix.size*4/1024/1024/1024:2.3f} GByte\")\ngc.collect()\nsparse_fix = scipy.sparse.csr_matrix(df_test_fix)\nsparse_X = scipy.sparse.vstack([sparse_X, sparse_fix])\n\ndel df_test_fix, sparse_fix\ngc.collect()\nprint(f\"Total sparse shape with fix: {str(sparse_X.shape):14} {sparse_X.size*4/1024/1024/1024:2.3f} GByte\")","metadata":{"execution":{"iopub.status.busy":"2022-10-22T11:56:27.258003Z","iopub.execute_input":"2022-10-22T11:56:27.259157Z","iopub.status.idle":"2022-10-22T11:56:42.413858Z","shell.execute_reply.started":"2022-10-22T11:56:27.259097Z","shell.execute_reply":"2022-10-22T11:56:42.412488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Apply the singular value decomposition.\nprint(f\"Shape of both before SVD: {sparse_X.shape}\")\nsvd = TruncatedSVD(n_components=512, random_state=1) # 512\nsparse_X = svd.fit_transform(sparse_X)\nprint(f\"Shape of both after SVD:  {sparse_X.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-10-08T15:30:07.665925Z","iopub.execute_input":"2022-10-08T15:30:07.666338Z","iopub.status.idle":"2022-10-08T15:58:47.795301Z","shell.execute_reply.started":"2022-10-08T15:30:07.666304Z","shell.execute_reply":"2022-10-08T15:58:47.793983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save result to file.\ndf_svd = pd.DataFrame(sparse_X)\ndf_svd.to_csv('svd.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T16:06:44.131465Z","iopub.execute_input":"2022-10-08T16:06:44.131921Z","iopub.status.idle":"2022-10-08T16:07:42.138713Z","shell.execute_reply.started":"2022-10-08T16:06:44.131883Z","shell.execute_reply":"2022-10-08T16:07:42.137527Z"},"trusted":true},"execution_count":null,"outputs":[]}]}