Clearing data

Author: comiam

projects/bm-news-classification/lab_1/nlp_1.ipynb

@@ -2,14 +2,10 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "initial_id",
    "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:44:33.991125465Z",
-     "start_time": "2024-01-14T18:44:33.689398819Z"
-    }
+    "collapsed": true
    },
    "outputs": [],
    "source": [
@@ -24,61 +20,22 @@
   },
   {
    "cell_type": "code",
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[nltk_data] Downloading package punkt to /home/maxim/nltk_data...\n",
-      "[nltk_data]   Package punkt is already up-to-date!\n",
-      "[nltk_data] Downloading package omw-1.4 to /home/maxim/nltk_data...\n",
-      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
-      "[nltk_data] Downloading package wordnet to /home/maxim/nltk_data...\n",
-      "[nltk_data]   Package wordnet is already up-to-date!\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": "True"
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "nltk.download('punkt')\n",
     "nltk.download('omw-1.4')\n",
     "nltk.download('wordnet')"
    ],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:44:34.341115279Z",
-     "start_time": "2024-01-14T18:44:33.992572197Z"
-    }
+    "collapsed": false
    },
    "id": "92a102ee9ba336d7",
-   "execution_count": 2
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "   label                                               text\n",
-      "0      3  Wall St. Bears Claw Back Into the Black (Reute...\n",
-      "1      3  Carlyle Looks Toward Commercial Aerospace (Reu...\n",
-      "2      3  Oil and Economy Cloud Stocks' Outlook (Reuters...\n",
-      "3      3  Iraq Halts Oil Exports from Main Southern Pipe...\n",
-      "4      3  Oil prices soar to all-time record, posing new...\n",
-      "Carlyle Looks Toward Commercial Aerospace (Reuters). Reuters - Private investment firm Carlyle Group,\\which has a reputation for making well-timed and occasionally\\controversial plays in the defense industry, has quietly placed\\its bets on another part of the market.\n"
-     ]
-    }
-   ],
+   "execution_count": null,
+   "outputs": [],
    "source": [
     "df = pd.read_csv(os.path.join('../dataset/raw/train.csv'), names=['label', 'Title', 'Description'])\n",
     "df['text'] = (df['Title'] + '. ' + df['Description'])\n",
@@ -87,42 +44,25 @@
     "print(df['text'][1])"
    ],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:44:34.665186382Z",
-     "start_time": "2024-01-14T18:44:34.342871597Z"
-    }
+    "collapsed": false
    },
    "id": "10d57caa4259352a"
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "0         3\n1         3\n2         3\n3         3\n4         3\n         ..\n119995    1\n119996    2\n119997    2\n119998    2\n119999    2\nName: label, Length: 120000, dtype: int64"
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "execution_count": null,
+   "outputs": [],
    "source": [
     "df['label']"
    ],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:44:34.673196662Z",
-     "start_time": "2024-01-14T18:44:34.667446876Z"
-    }
+    "collapsed": false
    },
    "id": "ad6c6f57643b7841"
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "outputs": [],
    "source": [
     "def process_file(df, path):\n",
@@ -147,17 +87,13 @@
     "        save_to_file(original, lemmatized, stemmed, os.path.join(str(row['label']), f'{str(counter)}.tsv'), path)"
    ],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:44:34.674441301Z",
-     "start_time": "2024-01-14T18:44:34.671605126Z"
-    }
+    "collapsed": false
    },
    "id": "b2f4a0464b92971e"
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "outputs": [],
    "source": [
     "df = pd.read_csv(os.path.join('../dataset/raw/train.csv'), names=['label', 'Title', 'Description'])\n",
@@ -167,17 +103,13 @@
     "process_file(df, os.path.join('..', 'assets', 'annotated-corpus', 'train'))"
    ],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:45:41.882656959Z",
-     "start_time": "2024-01-14T18:44:34.676441732Z"
-    }
+    "collapsed": false
    },
    "id": "9dc8c43fc3a82676"
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "outputs": [],
    "source": [
     "df = pd.read_csv(os.path.join('../dataset/raw/test.csv'), names=['label', 'Title', 'Description'])\n",
@@ -187,11 +119,7 @@
     "process_file(df, os.path.join('..', 'assets', 'annotated-corpus', 'test'))"
    ],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:45:46.209967327Z",
-     "start_time": "2024-01-14T18:45:41.882440048Z"
-    }
+    "collapsed": false
    },
    "id": "65ad549a2d70ff6c"
   },
@@ -200,14 +128,10 @@
    "outputs": [],
    "source": [],
    "metadata": {
-    "collapsed": false,
-    "ExecuteTime": {
-     "end_time": "2024-01-14T18:45:46.220037002Z",
-     "start_time": "2024-01-14T18:45:46.211685403Z"
-    }
+    "collapsed": false
    },
    "id": "b6b3ae49a9f95763",
-   "execution_count": 7
+   "execution_count": null
   },
   {
    "cell_type": "code",