Notebooks/Depression-Binary-Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0eb51d70-a544-4626-84e6-ee0bfcbf4a86",
   "metadata": {},
   "source": [
    "## Create Dataset (Only run when datasets are changed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9dd05219-a21a-406a-9d04-d5e7e4761372",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "753b1aab-d81d-436b-9855-55ef84abac92",
   "metadata": {},
   "outputs": [],
   "source": [
    "depression_data = pd.read_excel('../Data/Depression_Disorders_Data/depression_total_cleaned.xlsx')\n",
    "depression_data['labels'] = \"Depression\"\n",
    "\n",
    "normal_data = pd.read_excel('../Data/normal_dataset_new.xlsx').sample(30000)\n",
    "normal_data.drop('Text', axis=1, inplace=True)\n",
    "normal_data = normal_data.rename(columns={'Translated_Text': 'text'})\n",
    "normal_data['labels'] = 'Normal'\n",
    "\n",
    "dataset = pd.concat([depression_data, normal_data], axis=0)\n",
    "dataset = dataset.dropna().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e8d89ad2-2ba0-4bf5-8b6a-b358d5989af0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Çok yoğun ve düzensiz geçen günlerin ardından ...</td>\n",
       "      <td>Depression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gerçekten çok kötü hissediyorum ve sakin kalma...</td>\n",
       "      <td>Depression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Her ay, kendime bunun hormonlar olduğunu ve ge...</td>\n",
       "      <td>Depression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ağrım o kadar şiddetli ki bazen kendimi öldürm...</td>\n",
       "      <td>Depression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bu bebeği gerçekten istiyorum ama aynı zamanda...</td>\n",
       "      <td>Depression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57583</th>\n",
       "      <td>Bu olağanüstü bir akşam yemeği partisi olurdu,...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57584</th>\n",
       "      <td>Ağabeyim 28 yaşında ve ailenin en büyüğü kim?</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57585</th>\n",
       "      <td>Şimdi, bu çeki nakde çevirmeye ne dersiniz, ha...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57586</th>\n",
       "      <td>Her yönetmenle anlaşmazlık yaşadım. Ben 'Ve il...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57587</th>\n",
       "      <td>Nasıl göründüğünü merak ediyorum. Oh, onu görm...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>57588 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text      labels\n",
       "0      Çok yoğun ve düzensiz geçen günlerin ardından ...  Depression\n",
       "1      Gerçekten çok kötü hissediyorum ve sakin kalma...  Depression\n",
       "2      Her ay, kendime bunun hormonlar olduğunu ve ge...  Depression\n",
       "3      Ağrım o kadar şiddetli ki bazen kendimi öldürm...  Depression\n",
       "4      Bu bebeği gerçekten istiyorum ama aynı zamanda...  Depression\n",
       "...                                                  ...         ...\n",
       "57583  Bu olağanüstü bir akşam yemeği partisi olurdu,...      Normal\n",
       "57584      Ağabeyim 28 yaşında ve ailenin en büyüğü kim?      Normal\n",
       "57585  Şimdi, bu çeki nakde çevirmeye ne dersiniz, ha...      Normal\n",
       "57586  Her yönetmenle anlaşmazlık yaşadım. Ben 'Ve il...      Normal\n",
       "57587  Nasıl göründüğünü merak ediyorum. Oh, onu görm...      Normal\n",
       "\n",
       "[57588 rows x 2 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92b99c5a-b189-4f4a-9660-07d319517dd6",
   "metadata": {},
   "source": [
    "### Push Dataset to HuggingFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6c673a08-a9c8-4497-a98d-efd8bbf8494b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "train, test = train_test_split(dataset, test_size = 0.3, random_state=42)\n",
    "\n",
    "train_data = Dataset.from_pandas(train, preserve_index=False)\n",
    "test_data = Dataset.from_pandas(test, preserve_index=False)\n",
    "\n",
    "hg_data = DatasetDict({\n",
    "    \"train\": train_data,\n",
    "    \"test\": test_data\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4691c365-72f9-41b1-8336-34b298b086c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'labels'],\n",
       "        num_rows: 40311\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'labels'],\n",
       "        num_rows: 17277\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hg_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "dc2917ad-4260-4c27-a9d3-cceb691465e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
      "Token is valid (permission: write).\n",
      "Your token has been saved to C:\\Users\\halilibrahim.hatun\\.cache\\huggingface\\token\n",
      "Login successful\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e61e366549d243b29656b36bd3633751",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0da5248c872445a089660ce76c6aba8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/41 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7033442f382845ba9f5283cceb57fd96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e19e5b15bbe44c428b37e993aa0ce8af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/18 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data was pushed :)\n"
     ]
    }
   ],
   "source": [
    "!huggingface-cli login --token=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\n",
    "\n",
    "hg_data.push_to_hub(\"halilibr/dilbazlar-depression-binary-tr-dataset\")\n",
    "print(\"Data was pushed :)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c0b37429-bb5a-42ed-949f-dd33ea388788",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = hg_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8368ee72-0839-4e47-8da0-8b0627b0ee1a",
   "metadata": {},
   "source": [
    "## Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "be19ce61-5fb4-4333-8782-2ac6688e3a14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device:  cuda\n",
      "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n",
      "Token is valid (permission: write).\n",
      "Your token has been saved to C:\\Users\\halilibrahim.hatun\\.cache\\huggingface\\token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "import transformers\n",
    "from transformers import BertModel, BertTokenizer\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import glob\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Huggingface\n",
    "from datasets import load_dataset\n",
    "import huggingface_hub\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# specify GPU\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "print(\"Device: \", device)\n",
    "\n",
    "!huggingface-cli login --token=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"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b538893b-2b41-4a1c-b589-b387b59caa87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5206d57e4214869b9bea3c833aec479",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/40311 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "702d4f838c1647ea90c35fb01e8610a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/17277 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"dbmdz/bert-base-turkish-128k-uncased\")\n",
    "\n",
    "\n",
    "def tokenize_function(examples):\n",
    "   label_mapping = {\n",
    "        \"Normal\": 0,\n",
    "        \"Depression\": 1\n",
    "    }\n",
    "   inputs = tokenizer(examples[\"text\"], padding=\"max_length\", max_length=150, truncation=True)\n",
    "\n",
    "   labels = [label_mapping[label] for label in examples[\"labels\"]]\n",
    "   labels = torch.tensor(labels)\n",
    "\n",
    "   return {\n",
    "    \"input_ids\": inputs[\"input_ids\"],\n",
    "    \"attention_mask\": inputs[\"attention_mask\"],\n",
    "    \"labels\": labels,\n",
    "   }\n",
    "\n",
    "tokenized_dataset = dataset.map(tokenize_function, batched=True, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fa654b4c-f996-4dae-bb70-bd5c4c723082",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a601c10c-ca72-47db-b249-f3c103a3e52c",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Trainer Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a6ff5552-9481-4a15-a796-ca952a82e897",
   "metadata": {},
   "outputs": [],
   "source": [
    "id2label = {\n",
    "    0: \"Normal\",\n",
    "    1: \"Disorder\"\n",
    "}\n",
    "label2id = {\n",
    "    \"Normal\": 0,\n",
    "    \"Disorder\": 1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "474277ad-0375-4458-8998-50edb997a750",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at dbmdz/bert-base-turkish-128k-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForTokenClassification \n",
    "\n",
    "model = AutoModelForTokenClassification.from_pretrained(\"dbmdz/bert-base-turkish-128k-uncased\", num_labels=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2fb32c69-9eb9-4f93-b322-f6de256f17e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(output_dir=\"test_trainer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c09bf85-b898-4671-9cab-1927fdafce35",
   "metadata": {},
   "source": [
    "### Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "43c6e59c-30bb-4e5f-a009-32d702e91020",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import evaluate\n",
    "\n",
    "metric = evaluate.load(\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c87d11fc-1315-440f-ba85-40ec64c3b1d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f82af08e-cfa3-48ed-9f56-82b31031b12a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"temp_disorder_detect\",\n",
    "    learning_rate=0.00005,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=5,\n",
    "    weight_decay=0.01,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    load_best_model_at_end=True,\n",
    "    save_total_limit=3,\n",
    "    report_to='tensorboard'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "601baee0-3688-4e20-a03d-7c366af5bbe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_tokenized_dataset,\n",
    "    eval_dataset=test_tokenized_dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "cb4fdbb2-94f5-4a59-8d69-444a4264b53e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Expected input batch_size (4400) to match target batch_size (16).",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[64], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m trainer\u001b[38;5;241m.\u001b[39mtrain()\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\transformers\\trainer.py:1537\u001b[0m, in \u001b[0;36mTrainer.train\u001b[1;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[0;32m   1535\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[0;32m   1536\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1537\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m inner_training_loop(\n\u001b[0;32m   1538\u001b[0m         args\u001b[38;5;241m=\u001b[39margs,\n\u001b[0;32m   1539\u001b[0m         resume_from_checkpoint\u001b[38;5;241m=\u001b[39mresume_from_checkpoint,\n\u001b[0;32m   1540\u001b[0m         trial\u001b[38;5;241m=\u001b[39mtrial,\n\u001b[0;32m   1541\u001b[0m         ignore_keys_for_eval\u001b[38;5;241m=\u001b[39mignore_keys_for_eval,\n\u001b[0;32m   1542\u001b[0m     )\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\transformers\\trainer.py:1854\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[1;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[0;32m   1851\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_step_begin(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[0;32m   1853\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39maccumulate(model):\n\u001b[1;32m-> 1854\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_step(model, inputs)\n\u001b[0;32m   1856\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[0;32m   1857\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[0;32m   1858\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[0;32m   1859\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch\u001b[38;5;241m.\u001b[39misinf(tr_loss_step))\n\u001b[0;32m   1860\u001b[0m ):\n\u001b[0;32m   1861\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[0;32m   1862\u001b[0m     tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\transformers\\trainer.py:2735\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[1;34m(self, model, inputs)\u001b[0m\n\u001b[0;32m   2732\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m loss_mb\u001b[38;5;241m.\u001b[39mreduce_mean()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[0;32m   2734\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss_context_manager():\n\u001b[1;32m-> 2735\u001b[0m     loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss(model, inputs)\n\u001b[0;32m   2737\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mn_gpu \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m   2738\u001b[0m     loss \u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mmean()  \u001b[38;5;66;03m# mean() to average on multi-gpu parallel training\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\transformers\\trainer.py:2758\u001b[0m, in \u001b[0;36mTrainer.compute_loss\u001b[1;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[0;32m   2756\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   2757\u001b[0m     labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m-> 2758\u001b[0m outputs \u001b[38;5;241m=\u001b[39m model(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39minputs)\n\u001b[0;32m   2759\u001b[0m \u001b[38;5;66;03m# Save past state if it exists\u001b[39;00m\n\u001b[0;32m   2760\u001b[0m \u001b[38;5;66;03m# TODO: this needs to be fixed and made cleaner later.\u001b[39;00m\n\u001b[0;32m   2761\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mpast_index \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:1778\u001b[0m, in \u001b[0;36mBertForTokenClassification.forward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m   1776\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m labels \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m   1777\u001b[0m     loss_fct \u001b[38;5;241m=\u001b[39m CrossEntropyLoss()\n\u001b[1;32m-> 1778\u001b[0m     loss \u001b[38;5;241m=\u001b[39m loss_fct(logits\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_labels), labels\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))\n\u001b[0;32m   1780\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m return_dict:\n\u001b[0;32m   1781\u001b[0m     output \u001b[38;5;241m=\u001b[39m (logits,) \u001b[38;5;241m+\u001b[39m outputs[\u001b[38;5;241m2\u001b[39m:]\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\torch\\nn\\modules\\loss.py:1179\u001b[0m, in \u001b[0;36mCrossEntropyLoss.forward\u001b[1;34m(self, input, target)\u001b[0m\n\u001b[0;32m   1178\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor, target: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m-> 1179\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mcross_entropy(\u001b[38;5;28minput\u001b[39m, target, weight\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight,\n\u001b[0;32m   1180\u001b[0m                            ignore_index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mignore_index, reduction\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreduction,\n\u001b[0;32m   1181\u001b[0m                            label_smoothing\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlabel_smoothing)\n",
      "File \u001b[1;32m~\\AppData\\Local\\miniconda3\\envs\\psynexa_torch_cpu\\Lib\\site-packages\\torch\\nn\\functional.py:3053\u001b[0m, in \u001b[0;36mcross_entropy\u001b[1;34m(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)\u001b[0m\n\u001b[0;32m   3051\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m size_average \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m reduce \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m   3052\u001b[0m     reduction \u001b[38;5;241m=\u001b[39m _Reduction\u001b[38;5;241m.\u001b[39mlegacy_get_string(size_average, reduce)\n\u001b[1;32m-> 3053\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39m_C\u001b[38;5;241m.\u001b[39m_nn\u001b[38;5;241m.\u001b[39mcross_entropy_loss(\u001b[38;5;28minput\u001b[39m, target, weight, _Reduction\u001b[38;5;241m.\u001b[39mget_enum(reduction), ignore_index, label_smoothing)\n",
      "\u001b[1;31mValueError\u001b[0m: Expected input batch_size (4400) to match target batch_size (16)."
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0efa416d-9c57-40a5-8314-20d5ff5fb999",
   "metadata": {},
   "source": [
    "## Native Pytorch "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "07dd3f46-44da-4171-8921-32f0e8d57abc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_datasets = tokenized_dataset.remove_columns([\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6b4c8762-7ef3-4a51-9d64-09a134e55406",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_datasets.set_format(\"torch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2f97fe47-80eb-417c-8932-16d301c6ec34",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataloader = DataLoader(tokenized_datasets['train'], shuffle=True, batch_size=8)\n",
    "eval_dataloader = DataLoader(tokenized_datasets['test'], batch_size=8) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1e98e2bb-c0b8-41c6-a901-52581f747598",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at dbmdz/bert-base-turkish-128k-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"dbmdz/bert-base-turkish-128k-uncased\", num_labels=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "39b6a5e4-db59-40f4-8875-ef88a26487f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.optim import AdamW\n",
    "\n",
    "optimizer = AdamW(model.parameters(), lr=5e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4b4290c7-e44a-4d64-bb36-88b03065ff8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(128000, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import get_scheduler\n",
    "\n",
    "num_epochs = 3\n",
    "num_training_steps = num_epochs * len(train_dataloader)\n",
    "lr_scheduler = get_scheduler(\n",
    "    name=\"linear\", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps\n",
    ")\n",
    "\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4c6ac222-f1f6-4b36-ae5b-797a2850ca33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa17f62715e743328191cdca8f5f232b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15117 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "progress_bar = tqdm(range(num_training_steps))\n",
    "\n",
    "model.train()\n",
    "for epoch in range(num_epochs):\n",
    "    for batch in train_dataloader:\n",
    "        batch = {k: v.to(device) for k, v in batch.items()}\n",
    "        outputs = model(**batch)\n",
    "        loss = outputs.loss\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "        progress_bar.update(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "756a5fb6-522c-41a2-98a5-92070348b429",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.9839671239219772}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "metric = evaluate.load(\"accuracy\")\n",
    "model.eval()\n",
    "for batch in eval_dataloader:\n",
    "    batch = {k: v.to(device) for k, v in batch.items()}\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**batch)\n",
    "\n",
    "    logits = outputs.logits\n",
    "    predictions = torch.argmax(logits, dim=-1)\n",
    "    metric.add_batch(predictions=predictions, references=batch[\"labels\"])\n",
    "\n",
    "metric.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d6de1f4d-2da7-4f1f-b114-62dad61d5bf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa347415b4494ca1ac0461a3c2e17951",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/737M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/halilibr/dilbazlar-depression-binary-detection-model-acc-98.3/commit/f1cc3eb235bf33096e2f10a7cf8c3e86056ac849', commit_message='Upload BertForSequenceClassification', commit_description='', oid='f1cc3eb235bf33096e2f10a7cf8c3e86056ac849', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub('halilibr/dilbazlar-depression-binary-detection-model-acc-98.3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "bb6d7d3a-bc9f-4bc8-9ee6-9bf8e7f0b371",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/halilibr/dilbazlar-depression-binary-detection-model-acc-98.3/commit/7dbb0aaa49d1e295da2a83d8d917c3d73fe53768', commit_message='Upload tokenizer', commit_description='', oid='7dbb0aaa49d1e295da2a83d8d917c3d73fe53768', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub(\"halilibr/dilbazlar-depression-binary-detection-model-acc-98.3\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0e2af34-70a9-40ee-b16c-3fa58c3e4bdd",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f15139c6-bce8-439c-8ad9-a5b7b94020b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Text': 'Sosyal ortamlarda rahat bir şekilde iletişim kurabiliyorum.',\n",
       " 'Label': 'Normal'}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ba72fbbd-27da-48ff-90a7-fc67b4923ba3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f1e016d07ad477f870805646062b381",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/738 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1773b118a48640dc882162fe4a22e51a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/737M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9c2889f9b5644dbb9676b5b54f9bd0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/1.31k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17ccf6bf9f864f1bae96bd77ea7479c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/1.23M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e717045741fe4ce5b56e659ce54d2b6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/3.30M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dae5c616ef324781bdc873985d4a1edc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/132 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(128000, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "loaded_model = AutoModelForSequenceClassification.from_pretrained(\"halilibr/dilbazlar-binary-disorder-detection-model-acc-98.5\", num_labels=2)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"halilibr/dilbazlar-binary-disorder-detection-model-acc-98.5\")\n",
    "\n",
    "# Move the model to the appropriate device\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "loaded_model.to(device)\n",
    "\n",
    "# Ensure model is in evaluation mode\n",
    "loaded_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d08153a4-ad30-4c92-acf2-edaa24ff7163",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example input\n",
    "\n",
    "\n",
    "def predict(input_text):\n",
    "    # Tokenize the input (ensure the tokenizer is appropriate for your model)\n",
    "    inputs = tokenizer(input_text, max_length=150, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",
    "    \n",
    "    # Move the inputs to the appropriate device\n",
    "    inputs = {k: v.to(device) for k, v in inputs.items()}\n",
    "    \n",
    "    # Disable gradient computation for inference\n",
    "    with torch.no_grad():\n",
    "        # Forward pass to get outputs\n",
    "        outputs = loaded_model(**inputs)\n",
    "        \n",
    "        # Get the prediction\n",
    "        # Note: `AutoModel` might not include logits. Ensure you use the appropriate model class for your task.\n",
    "        if hasattr(outputs, 'logits'):\n",
    "            preds = torch.argmax(outputs.logits, dim=-1)\n",
    "        else:\n",
    "            # Handle the case where the model does not have logits (e.g., outputs are raw hidden states)\n",
    "            preds = torch.argmax(outputs[0], dim=-1)\n",
    "    \n",
    "    # Convert prediction to numpy array and print (if needed)\n",
    "    prediction = preds.cpu().numpy()[0]\n",
    "    print(outputs)\n",
    "    return prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bd85bc42-e661-4389-a2e8-10a34d070acc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SequenceClassifierOutput(loss=None, logits=tensor([[-0.8659,  1.5566]], device='cuda:0'), hidden_states=None, attentions=None)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text = \"geçen annemlerle oturuyorum\"\n",
    "\n",
    "predict(input_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f272375d-eecb-45fb-8b0f-d9aaf01b7df1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "daily_phrases = [\n",
    "    \"Bugün hava çok güzel.\",\n",
    "    \"İşe gitmem gerekiyor.\",\n",
    "    \"Kahvaltı yaptın mı?\",\n",
    "    \"Sinemaya gitmek ister misin?\",\n",
    "    \"Bu hafta sonu ne yapıyorsun?\",\n",
    "    \"Yeni bir kitap okuyorum.\",\n",
    "    \"Bu yemek harika olmuş.\",\n",
    "    \"Arkadaşlarla buluşacağım.\",\n",
    "    \"Yürüyüşe çıkalım mı?\",\n",
    "    \"Yeni bir hobi edindim.\",\n",
    "    \"Bu filmi izledin mi?\",\n",
    "    \"Tatilde nereye gitmek istersin?\",\n",
    "    \"Hangi sporu yapıyorsun?\",\n",
    "    \"Ders çalışmam gerekiyor.\",\n",
    "    \"Bu hafta sonu bir parti var.\",\n",
    "    \"Alışverişe çıkmak istiyorum.\",\n",
    "    \"Spor salonuna gidiyorum.\",\n",
    "    \"Yeni bir tarif denedim.\",\n",
    "    \"Bu hafta yoğun geçecek.\",\n",
    "    \"Yeni bir projeye başladım.\",\n",
    "    \"Yarın toplantım var.\",\n",
    "    \"Bahçede çalışmak istiyorum.\",\n",
    "    \"Yeni bir müzik albümü keşfettim.\",\n",
    "    \"Bu kitabı önerir misin?\",\n",
    "    \"Yarın sabah erken kalkmalıyım.\",\n",
    "    \"Ailemi ziyaret edeceğim.\",\n",
    "    \"Yeni bir dil öğreniyorum.\",\n",
    "    \"Bu dizi çok güzel.\",\n",
    "    \"Yoga yapmaya başladım.\",\n",
    "    \"İşe erken gitmem gerekiyor.\",\n",
    "    \"Bu hafta sonu dinlenmek istiyorum.\",\n",
    "    \"Yeni bir restoran denedim.\",\n",
    "    \"Hafta sonu planın nedir?\",\n",
    "    \"Yeni bir arkadaş edindim.\",\n",
    "    \"Bu film harikaydı.\",\n",
    "    \"Bisiklet sürmeye çıkacağım.\",\n",
    "    \"Bugün spor yapmalıyım.\",\n",
    "    \"Yeni bir bilgisayar aldım.\",\n",
    "    \"Bu kitabı bitirdim.\",\n",
    "    \"Yeni bir iş görüşmem var.\",\n",
    "    \"Evin dekorasyonunu değiştirdim.\",\n",
    "    \"Bu sabah yürüyüşe çıktım.\",\n",
    "    \"Yeni bir telefon aldım.\",\n",
    "    \"Bu hafta yoğun çalışacağım.\",\n",
    "    \"Hafta sonu tatile çıkıyorum.\",\n",
    "    \"Yeni bir proje üzerinde çalışıyorum.\",\n",
    "    \"İşler yolunda gidiyor.\",\n",
    "    \"Bu kitabı sevdim.\",\n",
    "    \"Yemek yapmayı çok seviyorum.\"\n",
    "]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e781a903-e06a-4f9f-bca6-08dd2a627441",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Psynexa Torch GPU (Python 3.11)",
   "language": "python",
   "name": "your_env_name"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}