transformer-distilation-pre.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "85a14d9b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:27:48.581944Z",
     "iopub.status.busy": "2023-04-15T11:27:48.581436Z",
     "iopub.status.idle": "2023-04-15T11:27:56.578418Z",
     "shell.execute_reply": "2023-04-15T11:27:56.576099Z"
    },
    "papermill": {
     "duration": 8.009994,
     "end_time": "2023-04-15T11:27:56.582635",
     "exception": false,
     "start_time": "2023-04-15T11:27:48.572641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import GPT2LMHeadModel, RobertaForMaskedLM, AutoModel, RobertaModel,RobertaConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "307cd4df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:27:56.596868Z",
     "iopub.status.busy": "2023-04-15T11:27:56.596357Z",
     "iopub.status.idle": "2023-04-15T11:28:05.340795Z",
     "shell.execute_reply": "2023-04-15T11:28:05.338034Z"
    },
    "papermill": {
     "duration": 8.755862,
     "end_time": "2023-04-15T11:28:05.344421",
     "exception": false,
     "start_time": "2023-04-15T11:27:56.588559",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at ../input/roberta-transformers-pytorch/roberta-base and are newly initialized: ['lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "roberta\n",
      "roberta.embeddings\n",
      "roberta.embeddings.word_embeddings\n",
      "roberta.embeddings.position_embeddings\n",
      "roberta.embeddings.token_type_embeddings\n",
      "roberta.embeddings.LayerNorm\n",
      "roberta.embeddings.dropout\n",
      "roberta.encoder\n",
      "roberta.encoder.layer\n",
      "roberta.encoder.layer.0\n",
      "roberta.encoder.layer.0.attention\n",
      "roberta.encoder.layer.0.attention.self\n",
      "roberta.encoder.layer.0.attention.self.query\n",
      "roberta.encoder.layer.0.attention.self.key\n",
      "roberta.encoder.layer.0.attention.self.value\n",
      "roberta.encoder.layer.0.attention.self.dropout\n",
      "roberta.encoder.layer.0.attention.output\n",
      "roberta.encoder.layer.0.attention.output.dense\n",
      "roberta.encoder.layer.0.attention.output.LayerNorm\n",
      "roberta.encoder.layer.0.attention.output.dropout\n",
      "roberta.encoder.layer.0.intermediate\n",
      "roberta.encoder.layer.0.intermediate.dense\n",
      "roberta.encoder.layer.0.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.0.output\n",
      "roberta.encoder.layer.0.output.dense\n",
      "roberta.encoder.layer.0.output.LayerNorm\n",
      "roberta.encoder.layer.0.output.dropout\n",
      "roberta.encoder.layer.1\n",
      "roberta.encoder.layer.1.attention\n",
      "roberta.encoder.layer.1.attention.self\n",
      "roberta.encoder.layer.1.attention.self.query\n",
      "roberta.encoder.layer.1.attention.self.key\n",
      "roberta.encoder.layer.1.attention.self.value\n",
      "roberta.encoder.layer.1.attention.self.dropout\n",
      "roberta.encoder.layer.1.attention.output\n",
      "roberta.encoder.layer.1.attention.output.dense\n",
      "roberta.encoder.layer.1.attention.output.LayerNorm\n",
      "roberta.encoder.layer.1.attention.output.dropout\n",
      "roberta.encoder.layer.1.intermediate\n",
      "roberta.encoder.layer.1.intermediate.dense\n",
      "roberta.encoder.layer.1.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.1.output\n",
      "roberta.encoder.layer.1.output.dense\n",
      "roberta.encoder.layer.1.output.LayerNorm\n",
      "roberta.encoder.layer.1.output.dropout\n",
      "roberta.encoder.layer.2\n",
      "roberta.encoder.layer.2.attention\n",
      "roberta.encoder.layer.2.attention.self\n",
      "roberta.encoder.layer.2.attention.self.query\n",
      "roberta.encoder.layer.2.attention.self.key\n",
      "roberta.encoder.layer.2.attention.self.value\n",
      "roberta.encoder.layer.2.attention.self.dropout\n",
      "roberta.encoder.layer.2.attention.output\n",
      "roberta.encoder.layer.2.attention.output.dense\n",
      "roberta.encoder.layer.2.attention.output.LayerNorm\n",
      "roberta.encoder.layer.2.attention.output.dropout\n",
      "roberta.encoder.layer.2.intermediate\n",
      "roberta.encoder.layer.2.intermediate.dense\n",
      "roberta.encoder.layer.2.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.2.output\n",
      "roberta.encoder.layer.2.output.dense\n",
      "roberta.encoder.layer.2.output.LayerNorm\n",
      "roberta.encoder.layer.2.output.dropout\n",
      "roberta.encoder.layer.3\n",
      "roberta.encoder.layer.3.attention\n",
      "roberta.encoder.layer.3.attention.self\n",
      "roberta.encoder.layer.3.attention.self.query\n",
      "roberta.encoder.layer.3.attention.self.key\n",
      "roberta.encoder.layer.3.attention.self.value\n",
      "roberta.encoder.layer.3.attention.self.dropout\n",
      "roberta.encoder.layer.3.attention.output\n",
      "roberta.encoder.layer.3.attention.output.dense\n",
      "roberta.encoder.layer.3.attention.output.LayerNorm\n",
      "roberta.encoder.layer.3.attention.output.dropout\n",
      "roberta.encoder.layer.3.intermediate\n",
      "roberta.encoder.layer.3.intermediate.dense\n",
      "roberta.encoder.layer.3.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.3.output\n",
      "roberta.encoder.layer.3.output.dense\n",
      "roberta.encoder.layer.3.output.LayerNorm\n",
      "roberta.encoder.layer.3.output.dropout\n",
      "roberta.encoder.layer.4\n",
      "roberta.encoder.layer.4.attention\n",
      "roberta.encoder.layer.4.attention.self\n",
      "roberta.encoder.layer.4.attention.self.query\n",
      "roberta.encoder.layer.4.attention.self.key\n",
      "roberta.encoder.layer.4.attention.self.value\n",
      "roberta.encoder.layer.4.attention.self.dropout\n",
      "roberta.encoder.layer.4.attention.output\n",
      "roberta.encoder.layer.4.attention.output.dense\n",
      "roberta.encoder.layer.4.attention.output.LayerNorm\n",
      "roberta.encoder.layer.4.attention.output.dropout\n",
      "roberta.encoder.layer.4.intermediate\n",
      "roberta.encoder.layer.4.intermediate.dense\n",
      "roberta.encoder.layer.4.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.4.output\n",
      "roberta.encoder.layer.4.output.dense\n",
      "roberta.encoder.layer.4.output.LayerNorm\n",
      "roberta.encoder.layer.4.output.dropout\n",
      "roberta.encoder.layer.5\n",
      "roberta.encoder.layer.5.attention\n",
      "roberta.encoder.layer.5.attention.self\n",
      "roberta.encoder.layer.5.attention.self.query\n",
      "roberta.encoder.layer.5.attention.self.key\n",
      "roberta.encoder.layer.5.attention.self.value\n",
      "roberta.encoder.layer.5.attention.self.dropout\n",
      "roberta.encoder.layer.5.attention.output\n",
      "roberta.encoder.layer.5.attention.output.dense\n",
      "roberta.encoder.layer.5.attention.output.LayerNorm\n",
      "roberta.encoder.layer.5.attention.output.dropout\n",
      "roberta.encoder.layer.5.intermediate\n",
      "roberta.encoder.layer.5.intermediate.dense\n",
      "roberta.encoder.layer.5.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.5.output\n",
      "roberta.encoder.layer.5.output.dense\n",
      "roberta.encoder.layer.5.output.LayerNorm\n",
      "roberta.encoder.layer.5.output.dropout\n",
      "roberta.encoder.layer.6\n",
      "roberta.encoder.layer.6.attention\n",
      "roberta.encoder.layer.6.attention.self\n",
      "roberta.encoder.layer.6.attention.self.query\n",
      "roberta.encoder.layer.6.attention.self.key\n",
      "roberta.encoder.layer.6.attention.self.value\n",
      "roberta.encoder.layer.6.attention.self.dropout\n",
      "roberta.encoder.layer.6.attention.output\n",
      "roberta.encoder.layer.6.attention.output.dense\n",
      "roberta.encoder.layer.6.attention.output.LayerNorm\n",
      "roberta.encoder.layer.6.attention.output.dropout\n",
      "roberta.encoder.layer.6.intermediate\n",
      "roberta.encoder.layer.6.intermediate.dense\n",
      "roberta.encoder.layer.6.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.6.output\n",
      "roberta.encoder.layer.6.output.dense\n",
      "roberta.encoder.layer.6.output.LayerNorm\n",
      "roberta.encoder.layer.6.output.dropout\n",
      "roberta.encoder.layer.7\n",
      "roberta.encoder.layer.7.attention\n",
      "roberta.encoder.layer.7.attention.self\n",
      "roberta.encoder.layer.7.attention.self.query\n",
      "roberta.encoder.layer.7.attention.self.key\n",
      "roberta.encoder.layer.7.attention.self.value\n",
      "roberta.encoder.layer.7.attention.self.dropout\n",
      "roberta.encoder.layer.7.attention.output\n",
      "roberta.encoder.layer.7.attention.output.dense\n",
      "roberta.encoder.layer.7.attention.output.LayerNorm\n",
      "roberta.encoder.layer.7.attention.output.dropout\n",
      "roberta.encoder.layer.7.intermediate\n",
      "roberta.encoder.layer.7.intermediate.dense\n",
      "roberta.encoder.layer.7.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.7.output\n",
      "roberta.encoder.layer.7.output.dense\n",
      "roberta.encoder.layer.7.output.LayerNorm\n",
      "roberta.encoder.layer.7.output.dropout\n",
      "roberta.encoder.layer.8\n",
      "roberta.encoder.layer.8.attention\n",
      "roberta.encoder.layer.8.attention.self\n",
      "roberta.encoder.layer.8.attention.self.query\n",
      "roberta.encoder.layer.8.attention.self.key\n",
      "roberta.encoder.layer.8.attention.self.value\n",
      "roberta.encoder.layer.8.attention.self.dropout\n",
      "roberta.encoder.layer.8.attention.output\n",
      "roberta.encoder.layer.8.attention.output.dense\n",
      "roberta.encoder.layer.8.attention.output.LayerNorm\n",
      "roberta.encoder.layer.8.attention.output.dropout\n",
      "roberta.encoder.layer.8.intermediate\n",
      "roberta.encoder.layer.8.intermediate.dense\n",
      "roberta.encoder.layer.8.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.8.output\n",
      "roberta.encoder.layer.8.output.dense\n",
      "roberta.encoder.layer.8.output.LayerNorm\n",
      "roberta.encoder.layer.8.output.dropout\n",
      "roberta.encoder.layer.9\n",
      "roberta.encoder.layer.9.attention\n",
      "roberta.encoder.layer.9.attention.self\n",
      "roberta.encoder.layer.9.attention.self.query\n",
      "roberta.encoder.layer.9.attention.self.key\n",
      "roberta.encoder.layer.9.attention.self.value\n",
      "roberta.encoder.layer.9.attention.self.dropout\n",
      "roberta.encoder.layer.9.attention.output\n",
      "roberta.encoder.layer.9.attention.output.dense\n",
      "roberta.encoder.layer.9.attention.output.LayerNorm\n",
      "roberta.encoder.layer.9.attention.output.dropout\n",
      "roberta.encoder.layer.9.intermediate\n",
      "roberta.encoder.layer.9.intermediate.dense\n",
      "roberta.encoder.layer.9.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.9.output\n",
      "roberta.encoder.layer.9.output.dense\n",
      "roberta.encoder.layer.9.output.LayerNorm\n",
      "roberta.encoder.layer.9.output.dropout\n",
      "roberta.encoder.layer.10\n",
      "roberta.encoder.layer.10.attention\n",
      "roberta.encoder.layer.10.attention.self\n",
      "roberta.encoder.layer.10.attention.self.query\n",
      "roberta.encoder.layer.10.attention.self.key\n",
      "roberta.encoder.layer.10.attention.self.value\n",
      "roberta.encoder.layer.10.attention.self.dropout\n",
      "roberta.encoder.layer.10.attention.output\n",
      "roberta.encoder.layer.10.attention.output.dense\n",
      "roberta.encoder.layer.10.attention.output.LayerNorm\n",
      "roberta.encoder.layer.10.attention.output.dropout\n",
      "roberta.encoder.layer.10.intermediate\n",
      "roberta.encoder.layer.10.intermediate.dense\n",
      "roberta.encoder.layer.10.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.10.output\n",
      "roberta.encoder.layer.10.output.dense\n",
      "roberta.encoder.layer.10.output.LayerNorm\n",
      "roberta.encoder.layer.10.output.dropout\n",
      "roberta.encoder.layer.11\n",
      "roberta.encoder.layer.11.attention\n",
      "roberta.encoder.layer.11.attention.self\n",
      "roberta.encoder.layer.11.attention.self.query\n",
      "roberta.encoder.layer.11.attention.self.key\n",
      "roberta.encoder.layer.11.attention.self.value\n",
      "roberta.encoder.layer.11.attention.self.dropout\n",
      "roberta.encoder.layer.11.attention.output\n",
      "roberta.encoder.layer.11.attention.output.dense\n",
      "roberta.encoder.layer.11.attention.output.LayerNorm\n",
      "roberta.encoder.layer.11.attention.output.dropout\n",
      "roberta.encoder.layer.11.intermediate\n",
      "roberta.encoder.layer.11.intermediate.dense\n",
      "roberta.encoder.layer.11.intermediate.intermediate_act_fn\n",
      "roberta.encoder.layer.11.output\n",
      "roberta.encoder.layer.11.output.dense\n",
      "roberta.encoder.layer.11.output.LayerNorm\n",
      "roberta.encoder.layer.11.output.dropout\n",
      "lm_head\n",
      "lm_head.dense\n",
      "lm_head.layer_norm\n",
      "lm_head.decoder\n"
     ]
    }
   ],
   "source": [
    "model = RobertaForMaskedLM.from_pretrained(\"../input/roberta-transformers-pytorch/roberta-base\")\n",
    "for name, layer in model.named_modules():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c8c2ecc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:05.358761Z",
     "iopub.status.busy": "2023-04-15T11:28:05.358299Z",
     "iopub.status.idle": "2023-04-15T11:28:07.667761Z",
     "shell.execute_reply": "2023-04-15T11:28:07.664902Z"
    },
    "papermill": {
     "duration": 2.321045,
     "end_time": "2023-04-15T11:28:07.671729",
     "exception": false,
     "start_time": "2023-04-15T11:28:05.350684",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "embeddings\n",
      "embeddings.word_embeddings\n",
      "embeddings.position_embeddings\n",
      "embeddings.token_type_embeddings\n",
      "embeddings.LayerNorm\n",
      "embeddings.dropout\n",
      "encoder\n",
      "encoder.layer\n",
      "encoder.layer.0\n",
      "encoder.layer.0.attention\n",
      "encoder.layer.0.attention.self\n",
      "encoder.layer.0.attention.self.query\n",
      "encoder.layer.0.attention.self.key\n",
      "encoder.layer.0.attention.self.value\n",
      "encoder.layer.0.attention.self.dropout\n",
      "encoder.layer.0.attention.output\n",
      "encoder.layer.0.attention.output.dense\n",
      "encoder.layer.0.attention.output.LayerNorm\n",
      "encoder.layer.0.attention.output.dropout\n",
      "encoder.layer.0.intermediate\n",
      "encoder.layer.0.intermediate.dense\n",
      "encoder.layer.0.intermediate.intermediate_act_fn\n",
      "encoder.layer.0.output\n",
      "encoder.layer.0.output.dense\n",
      "encoder.layer.0.output.LayerNorm\n",
      "encoder.layer.0.output.dropout\n",
      "encoder.layer.1\n",
      "encoder.layer.1.attention\n",
      "encoder.layer.1.attention.self\n",
      "encoder.layer.1.attention.self.query\n",
      "encoder.layer.1.attention.self.key\n",
      "encoder.layer.1.attention.self.value\n",
      "encoder.layer.1.attention.self.dropout\n",
      "encoder.layer.1.attention.output\n",
      "encoder.layer.1.attention.output.dense\n",
      "encoder.layer.1.attention.output.LayerNorm\n",
      "encoder.layer.1.attention.output.dropout\n",
      "encoder.layer.1.intermediate\n",
      "encoder.layer.1.intermediate.dense\n",
      "encoder.layer.1.intermediate.intermediate_act_fn\n",
      "encoder.layer.1.output\n",
      "encoder.layer.1.output.dense\n",
      "encoder.layer.1.output.LayerNorm\n",
      "encoder.layer.1.output.dropout\n",
      "encoder.layer.2\n",
      "encoder.layer.2.attention\n",
      "encoder.layer.2.attention.self\n",
      "encoder.layer.2.attention.self.query\n",
      "encoder.layer.2.attention.self.key\n",
      "encoder.layer.2.attention.self.value\n",
      "encoder.layer.2.attention.self.dropout\n",
      "encoder.layer.2.attention.output\n",
      "encoder.layer.2.attention.output.dense\n",
      "encoder.layer.2.attention.output.LayerNorm\n",
      "encoder.layer.2.attention.output.dropout\n",
      "encoder.layer.2.intermediate\n",
      "encoder.layer.2.intermediate.dense\n",
      "encoder.layer.2.intermediate.intermediate_act_fn\n",
      "encoder.layer.2.output\n",
      "encoder.layer.2.output.dense\n",
      "encoder.layer.2.output.LayerNorm\n",
      "encoder.layer.2.output.dropout\n",
      "encoder.layer.3\n",
      "encoder.layer.3.attention\n",
      "encoder.layer.3.attention.self\n",
      "encoder.layer.3.attention.self.query\n",
      "encoder.layer.3.attention.self.key\n",
      "encoder.layer.3.attention.self.value\n",
      "encoder.layer.3.attention.self.dropout\n",
      "encoder.layer.3.attention.output\n",
      "encoder.layer.3.attention.output.dense\n",
      "encoder.layer.3.attention.output.LayerNorm\n",
      "encoder.layer.3.attention.output.dropout\n",
      "encoder.layer.3.intermediate\n",
      "encoder.layer.3.intermediate.dense\n",
      "encoder.layer.3.intermediate.intermediate_act_fn\n",
      "encoder.layer.3.output\n",
      "encoder.layer.3.output.dense\n",
      "encoder.layer.3.output.LayerNorm\n",
      "encoder.layer.3.output.dropout\n",
      "encoder.layer.4\n",
      "encoder.layer.4.attention\n",
      "encoder.layer.4.attention.self\n",
      "encoder.layer.4.attention.self.query\n",
      "encoder.layer.4.attention.self.key\n",
      "encoder.layer.4.attention.self.value\n",
      "encoder.layer.4.attention.self.dropout\n",
      "encoder.layer.4.attention.output\n",
      "encoder.layer.4.attention.output.dense\n",
      "encoder.layer.4.attention.output.LayerNorm\n",
      "encoder.layer.4.attention.output.dropout\n",
      "encoder.layer.4.intermediate\n",
      "encoder.layer.4.intermediate.dense\n",
      "encoder.layer.4.intermediate.intermediate_act_fn\n",
      "encoder.layer.4.output\n",
      "encoder.layer.4.output.dense\n",
      "encoder.layer.4.output.LayerNorm\n",
      "encoder.layer.4.output.dropout\n",
      "encoder.layer.5\n",
      "encoder.layer.5.attention\n",
      "encoder.layer.5.attention.self\n",
      "encoder.layer.5.attention.self.query\n",
      "encoder.layer.5.attention.self.key\n",
      "encoder.layer.5.attention.self.value\n",
      "encoder.layer.5.attention.self.dropout\n",
      "encoder.layer.5.attention.output\n",
      "encoder.layer.5.attention.output.dense\n",
      "encoder.layer.5.attention.output.LayerNorm\n",
      "encoder.layer.5.attention.output.dropout\n",
      "encoder.layer.5.intermediate\n",
      "encoder.layer.5.intermediate.dense\n",
      "encoder.layer.5.intermediate.intermediate_act_fn\n",
      "encoder.layer.5.output\n",
      "encoder.layer.5.output.dense\n",
      "encoder.layer.5.output.LayerNorm\n",
      "encoder.layer.5.output.dropout\n",
      "encoder.layer.6\n",
      "encoder.layer.6.attention\n",
      "encoder.layer.6.attention.self\n",
      "encoder.layer.6.attention.self.query\n",
      "encoder.layer.6.attention.self.key\n",
      "encoder.layer.6.attention.self.value\n",
      "encoder.layer.6.attention.self.dropout\n",
      "encoder.layer.6.attention.output\n",
      "encoder.layer.6.attention.output.dense\n",
      "encoder.layer.6.attention.output.LayerNorm\n",
      "encoder.layer.6.attention.output.dropout\n",
      "encoder.layer.6.intermediate\n",
      "encoder.layer.6.intermediate.dense\n",
      "encoder.layer.6.intermediate.intermediate_act_fn\n",
      "encoder.layer.6.output\n",
      "encoder.layer.6.output.dense\n",
      "encoder.layer.6.output.LayerNorm\n",
      "encoder.layer.6.output.dropout\n",
      "encoder.layer.7\n",
      "encoder.layer.7.attention\n",
      "encoder.layer.7.attention.self\n",
      "encoder.layer.7.attention.self.query\n",
      "encoder.layer.7.attention.self.key\n",
      "encoder.layer.7.attention.self.value\n",
      "encoder.layer.7.attention.self.dropout\n",
      "encoder.layer.7.attention.output\n",
      "encoder.layer.7.attention.output.dense\n",
      "encoder.layer.7.attention.output.LayerNorm\n",
      "encoder.layer.7.attention.output.dropout\n",
      "encoder.layer.7.intermediate\n",
      "encoder.layer.7.intermediate.dense\n",
      "encoder.layer.7.intermediate.intermediate_act_fn\n",
      "encoder.layer.7.output\n",
      "encoder.layer.7.output.dense\n",
      "encoder.layer.7.output.LayerNorm\n",
      "encoder.layer.7.output.dropout\n",
      "encoder.layer.8\n",
      "encoder.layer.8.attention\n",
      "encoder.layer.8.attention.self\n",
      "encoder.layer.8.attention.self.query\n",
      "encoder.layer.8.attention.self.key\n",
      "encoder.layer.8.attention.self.value\n",
      "encoder.layer.8.attention.self.dropout\n",
      "encoder.layer.8.attention.output\n",
      "encoder.layer.8.attention.output.dense\n",
      "encoder.layer.8.attention.output.LayerNorm\n",
      "encoder.layer.8.attention.output.dropout\n",
      "encoder.layer.8.intermediate\n",
      "encoder.layer.8.intermediate.dense\n",
      "encoder.layer.8.intermediate.intermediate_act_fn\n",
      "encoder.layer.8.output\n",
      "encoder.layer.8.output.dense\n",
      "encoder.layer.8.output.LayerNorm\n",
      "encoder.layer.8.output.dropout\n",
      "encoder.layer.9\n",
      "encoder.layer.9.attention\n",
      "encoder.layer.9.attention.self\n",
      "encoder.layer.9.attention.self.query\n",
      "encoder.layer.9.attention.self.key\n",
      "encoder.layer.9.attention.self.value\n",
      "encoder.layer.9.attention.self.dropout\n",
      "encoder.layer.9.attention.output\n",
      "encoder.layer.9.attention.output.dense\n",
      "encoder.layer.9.attention.output.LayerNorm\n",
      "encoder.layer.9.attention.output.dropout\n",
      "encoder.layer.9.intermediate\n",
      "encoder.layer.9.intermediate.dense\n",
      "encoder.layer.9.intermediate.intermediate_act_fn\n",
      "encoder.layer.9.output\n",
      "encoder.layer.9.output.dense\n",
      "encoder.layer.9.output.LayerNorm\n",
      "encoder.layer.9.output.dropout\n",
      "encoder.layer.10\n",
      "encoder.layer.10.attention\n",
      "encoder.layer.10.attention.self\n",
      "encoder.layer.10.attention.self.query\n",
      "encoder.layer.10.attention.self.key\n",
      "encoder.layer.10.attention.self.value\n",
      "encoder.layer.10.attention.self.dropout\n",
      "encoder.layer.10.attention.output\n",
      "encoder.layer.10.attention.output.dense\n",
      "encoder.layer.10.attention.output.LayerNorm\n",
      "encoder.layer.10.attention.output.dropout\n",
      "encoder.layer.10.intermediate\n",
      "encoder.layer.10.intermediate.dense\n",
      "encoder.layer.10.intermediate.intermediate_act_fn\n",
      "encoder.layer.10.output\n",
      "encoder.layer.10.output.dense\n",
      "encoder.layer.10.output.LayerNorm\n",
      "encoder.layer.10.output.dropout\n",
      "encoder.layer.11\n",
      "encoder.layer.11.attention\n",
      "encoder.layer.11.attention.self\n",
      "encoder.layer.11.attention.self.query\n",
      "encoder.layer.11.attention.self.key\n",
      "encoder.layer.11.attention.self.value\n",
      "encoder.layer.11.attention.self.dropout\n",
      "encoder.layer.11.attention.output\n",
      "encoder.layer.11.attention.output.dense\n",
      "encoder.layer.11.attention.output.LayerNorm\n",
      "encoder.layer.11.attention.output.dropout\n",
      "encoder.layer.11.intermediate\n",
      "encoder.layer.11.intermediate.dense\n",
      "encoder.layer.11.intermediate.intermediate_act_fn\n",
      "encoder.layer.11.output\n",
      "encoder.layer.11.output.dense\n",
      "encoder.layer.11.output.LayerNorm\n",
      "encoder.layer.11.output.dropout\n",
      "pooler\n",
      "pooler.dense\n",
      "pooler.activation\n"
     ]
    }
   ],
   "source": [
    "model = RobertaModel.from_pretrained(\"../input/roberta-transformers-pytorch/roberta-base\")\n",
    "for name, layer in model.named_modules():\n",
    "    print(name)\n",
    "state_dict = model.state_dict()\n",
    "compressed_sd = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ccd7676d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:07.687105Z",
     "iopub.status.busy": "2023-04-15T11:28:07.686567Z",
     "iopub.status.idle": "2023-04-15T11:28:07.695090Z",
     "shell.execute_reply": "2023-04-15T11:28:07.693083Z"
    },
    "papermill": {
     "duration": 0.019645,
     "end_time": "2023-04-15T11:28:07.698097",
     "exception": false,
     "start_time": "2023-04-15T11:28:07.678452",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#  prefix = \"roberta\"\n",
    "compressed_sd['embeddings.position_ids'] = state_dict['embeddings.position_ids']\n",
    "for w in [\"word_embeddings\", \"position_embeddings\", \"token_type_embeddings\"]:\n",
    "    param_name = f\"embeddings.{w}.weight\"\n",
    "    compressed_sd[param_name] = state_dict[param_name]\n",
    "for w in [\"weight\", \"bias\"]:\n",
    "    param_name = f\"embeddings.LayerNorm.{w}\"\n",
    "    compressed_sd[param_name] = state_dict[param_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3b3abb26",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:07.713205Z",
     "iopub.status.busy": "2023-04-15T11:28:07.712402Z",
     "iopub.status.idle": "2023-04-15T11:28:07.719429Z",
     "shell.execute_reply": "2023-04-15T11:28:07.718098Z"
    },
    "papermill": {
     "duration": 0.018196,
     "end_time": "2023-04-15T11:28:07.722548",
     "exception": false,
     "start_time": "2023-04-15T11:28:07.704352",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "std_idx = 0\n",
    "for teacher_idx in [0, 2, 4, 7, 9, 11]:\n",
    "    for layer in [\"attention.self.query\",\"attention.self.key\",\"attention.self.value\",\"attention.output.dense\",\"attention.output.LayerNorm\",\"intermediate.dense\",\"output.dense\",\"output.LayerNorm\"]:\n",
    "        for w in [\"weight\", \"bias\"]:\n",
    "            compressed_sd[f\"encoder.layer.{std_idx}.{layer}.{w}\"] = state_dict[f\"encoder.layer.{teacher_idx}.{layer}.{w}\"]\n",
    "    std_idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9aea504e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:07.740053Z",
     "iopub.status.busy": "2023-04-15T11:28:07.738236Z",
     "iopub.status.idle": "2023-04-15T11:28:07.746295Z",
     "shell.execute_reply": "2023-04-15T11:28:07.744836Z"
    },
    "papermill": {
     "duration": 0.020428,
     "end_time": "2023-04-15T11:28:07.749595",
     "exception": false,
     "start_time": "2023-04-15T11:28:07.729167",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for layer in [\"pooler.dense.weight\", \"pooler.dense.bias\"]:\n",
    "    compressed_sd[f\"{layer}\"] = state_dict[f\"{layer}\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "20292060",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:07.764244Z",
     "iopub.status.busy": "2023-04-15T11:28:07.763745Z",
     "iopub.status.idle": "2023-04-15T11:28:07.771512Z",
     "shell.execute_reply": "2023-04-15T11:28:07.769622Z"
    },
    "papermill": {
     "duration": 0.019075,
     "end_time": "2023-04-15T11:28:07.774880",
     "exception": false,
     "start_time": "2023-04-15T11:28:07.755805",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of params transferred for distillation: 104\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of params transferred for distillation: {len(compressed_sd.keys())}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5942cb72",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:07.789546Z",
     "iopub.status.busy": "2023-04-15T11:28:07.789078Z",
     "iopub.status.idle": "2023-04-15T11:28:07.798731Z",
     "shell.execute_reply": "2023-04-15T11:28:07.797349Z"
    },
    "papermill": {
     "duration": 0.020735,
     "end_time": "2023-04-15T11:28:07.801868",
     "exception": false,
     "start_time": "2023-04-15T11:28:07.781133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['embeddings.position_ids', 'embeddings.word_embeddings.weight', 'embeddings.position_embeddings.weight', 'embeddings.token_type_embeddings.weight', 'embeddings.LayerNorm.weight', 'embeddings.LayerNorm.bias', 'encoder.layer.0.attention.self.query.weight', 'encoder.layer.0.attention.self.query.bias', 'encoder.layer.0.attention.self.key.weight', 'encoder.layer.0.attention.self.key.bias', 'encoder.layer.0.attention.self.value.weight', 'encoder.layer.0.attention.self.value.bias', 'encoder.layer.0.attention.output.dense.weight', 'encoder.layer.0.attention.output.dense.bias', 'encoder.layer.0.attention.output.LayerNorm.weight', 'encoder.layer.0.attention.output.LayerNorm.bias', 'encoder.layer.0.intermediate.dense.weight', 'encoder.layer.0.intermediate.dense.bias', 'encoder.layer.0.output.dense.weight', 'encoder.layer.0.output.dense.bias', 'encoder.layer.0.output.LayerNorm.weight', 'encoder.layer.0.output.LayerNorm.bias', 'encoder.layer.1.attention.self.query.weight', 'encoder.layer.1.attention.self.query.bias', 'encoder.layer.1.attention.self.key.weight', 'encoder.layer.1.attention.self.key.bias', 'encoder.layer.1.attention.self.value.weight', 'encoder.layer.1.attention.self.value.bias', 'encoder.layer.1.attention.output.dense.weight', 'encoder.layer.1.attention.output.dense.bias', 'encoder.layer.1.attention.output.LayerNorm.weight', 'encoder.layer.1.attention.output.LayerNorm.bias', 'encoder.layer.1.intermediate.dense.weight', 'encoder.layer.1.intermediate.dense.bias', 'encoder.layer.1.output.dense.weight', 'encoder.layer.1.output.dense.bias', 'encoder.layer.1.output.LayerNorm.weight', 'encoder.layer.1.output.LayerNorm.bias', 'encoder.layer.2.attention.self.query.weight', 'encoder.layer.2.attention.self.query.bias', 'encoder.layer.2.attention.self.key.weight', 'encoder.layer.2.attention.self.key.bias', 'encoder.layer.2.attention.self.value.weight', 'encoder.layer.2.attention.self.value.bias', 'encoder.layer.2.attention.output.dense.weight', 'encoder.layer.2.attention.output.dense.bias', 'encoder.layer.2.attention.output.LayerNorm.weight', 'encoder.layer.2.attention.output.LayerNorm.bias', 'encoder.layer.2.intermediate.dense.weight', 'encoder.layer.2.intermediate.dense.bias', 'encoder.layer.2.output.dense.weight', 'encoder.layer.2.output.dense.bias', 'encoder.layer.2.output.LayerNorm.weight', 'encoder.layer.2.output.LayerNorm.bias', 'encoder.layer.3.attention.self.query.weight', 'encoder.layer.3.attention.self.query.bias', 'encoder.layer.3.attention.self.key.weight', 'encoder.layer.3.attention.self.key.bias', 'encoder.layer.3.attention.self.value.weight', 'encoder.layer.3.attention.self.value.bias', 'encoder.layer.3.attention.output.dense.weight', 'encoder.layer.3.attention.output.dense.bias', 'encoder.layer.3.attention.output.LayerNorm.weight', 'encoder.layer.3.attention.output.LayerNorm.bias', 'encoder.layer.3.intermediate.dense.weight', 'encoder.layer.3.intermediate.dense.bias', 'encoder.layer.3.output.dense.weight', 'encoder.layer.3.output.dense.bias', 'encoder.layer.3.output.LayerNorm.weight', 'encoder.layer.3.output.LayerNorm.bias', 'encoder.layer.4.attention.self.query.weight', 'encoder.layer.4.attention.self.query.bias', 'encoder.layer.4.attention.self.key.weight', 'encoder.layer.4.attention.self.key.bias', 'encoder.layer.4.attention.self.value.weight', 'encoder.layer.4.attention.self.value.bias', 'encoder.layer.4.attention.output.dense.weight', 'encoder.layer.4.attention.output.dense.bias', 'encoder.layer.4.attention.output.LayerNorm.weight', 'encoder.layer.4.attention.output.LayerNorm.bias', 'encoder.layer.4.intermediate.dense.weight', 'encoder.layer.4.intermediate.dense.bias', 'encoder.layer.4.output.dense.weight', 'encoder.layer.4.output.dense.bias', 'encoder.layer.4.output.LayerNorm.weight', 'encoder.layer.4.output.LayerNorm.bias', 'encoder.layer.5.attention.self.query.weight', 'encoder.layer.5.attention.self.query.bias', 'encoder.layer.5.attention.self.key.weight', 'encoder.layer.5.attention.self.key.bias', 'encoder.layer.5.attention.self.value.weight', 'encoder.layer.5.attention.self.value.bias', 'encoder.layer.5.attention.output.dense.weight', 'encoder.layer.5.attention.output.dense.bias', 'encoder.layer.5.attention.output.LayerNorm.weight', 'encoder.layer.5.attention.output.LayerNorm.bias', 'encoder.layer.5.intermediate.dense.weight', 'encoder.layer.5.intermediate.dense.bias', 'encoder.layer.5.output.dense.weight', 'encoder.layer.5.output.dense.bias', 'encoder.layer.5.output.LayerNorm.weight', 'encoder.layer.5.output.LayerNorm.bias', 'pooler.dense.weight', 'pooler.dense.bias'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compressed_sd.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "916ad498",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:07.816628Z",
     "iopub.status.busy": "2023-04-15T11:28:07.816171Z",
     "iopub.status.idle": "2023-04-15T11:28:08.298921Z",
     "shell.execute_reply": "2023-04-15T11:28:08.297601Z"
    },
    "papermill": {
     "duration": 0.494208,
     "end_time": "2023-04-15T11:28:08.302449",
     "exception": false,
     "start_time": "2023-04-15T11:28:07.808241",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.save(compressed_sd, 'roberta_base_6layers.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bfc94d80",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:08.318706Z",
     "iopub.status.busy": "2023-04-15T11:28:08.318153Z",
     "iopub.status.idle": "2023-04-15T11:28:08.654996Z",
     "shell.execute_reply": "2023-04-15T11:28:08.653547Z"
    },
    "papermill": {
     "duration": 0.349441,
     "end_time": "2023-04-15T11:28:08.658394",
     "exception": false,
     "start_time": "2023-04-15T11:28:08.308953",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1b7f9ab3e594f109ab13a217f258584",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/481 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "config = RobertaConfig.from_pretrained('roberta-base',num_hidden_layers = 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5b3638c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:08.675187Z",
     "iopub.status.busy": "2023-04-15T11:28:08.674645Z",
     "iopub.status.idle": "2023-04-15T11:28:08.685222Z",
     "shell.execute_reply": "2023-04-15T11:28:08.683554Z"
    },
    "papermill": {
     "duration": 0.023067,
     "end_time": "2023-04-15T11:28:08.688760",
     "exception": false,
     "start_time": "2023-04-15T11:28:08.665693",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobertaConfig {\n",
       "  \"architectures\": [\n",
       "    \"RobertaForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"bos_token_id\": 0,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"eos_token_id\": 2,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-05,\n",
       "  \"max_position_embeddings\": 514,\n",
       "  \"model_type\": \"roberta\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 6,\n",
       "  \"pad_token_id\": 1,\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.27.4\",\n",
       "  \"type_vocab_size\": 1,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 50265\n",
       "}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0723a21b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:08.704849Z",
     "iopub.status.busy": "2023-04-15T11:28:08.704377Z",
     "iopub.status.idle": "2023-04-15T11:28:10.373419Z",
     "shell.execute_reply": "2023-04-15T11:28:10.371530Z"
    },
    "papermill": {
     "duration": 1.681909,
     "end_time": "2023-04-15T11:28:10.377694",
     "exception": false,
     "start_time": "2023-04-15T11:28:08.695785",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_student = AutoModel.from_config(config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5fabdcb8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:10.400157Z",
     "iopub.status.busy": "2023-04-15T11:28:10.399608Z",
     "iopub.status.idle": "2023-04-15T11:28:10.409506Z",
     "shell.execute_reply": "2023-04-15T11:28:10.408061Z"
    },
    "papermill": {
     "duration": 0.023088,
     "end_time": "2023-04-15T11:28:10.413095",
     "exception": false,
     "start_time": "2023-04-15T11:28:10.390007",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "embeddings\n",
      "embeddings.word_embeddings\n",
      "embeddings.position_embeddings\n",
      "embeddings.token_type_embeddings\n",
      "embeddings.LayerNorm\n",
      "embeddings.dropout\n",
      "encoder\n",
      "encoder.layer\n",
      "encoder.layer.0\n",
      "encoder.layer.0.attention\n",
      "encoder.layer.0.attention.self\n",
      "encoder.layer.0.attention.self.query\n",
      "encoder.layer.0.attention.self.key\n",
      "encoder.layer.0.attention.self.value\n",
      "encoder.layer.0.attention.self.dropout\n",
      "encoder.layer.0.attention.output\n",
      "encoder.layer.0.attention.output.dense\n",
      "encoder.layer.0.attention.output.LayerNorm\n",
      "encoder.layer.0.attention.output.dropout\n",
      "encoder.layer.0.intermediate\n",
      "encoder.layer.0.intermediate.dense\n",
      "encoder.layer.0.intermediate.intermediate_act_fn\n",
      "encoder.layer.0.output\n",
      "encoder.layer.0.output.dense\n",
      "encoder.layer.0.output.LayerNorm\n",
      "encoder.layer.0.output.dropout\n",
      "encoder.layer.1\n",
      "encoder.layer.1.attention\n",
      "encoder.layer.1.attention.self\n",
      "encoder.layer.1.attention.self.query\n",
      "encoder.layer.1.attention.self.key\n",
      "encoder.layer.1.attention.self.value\n",
      "encoder.layer.1.attention.self.dropout\n",
      "encoder.layer.1.attention.output\n",
      "encoder.layer.1.attention.output.dense\n",
      "encoder.layer.1.attention.output.LayerNorm\n",
      "encoder.layer.1.attention.output.dropout\n",
      "encoder.layer.1.intermediate\n",
      "encoder.layer.1.intermediate.dense\n",
      "encoder.layer.1.intermediate.intermediate_act_fn\n",
      "encoder.layer.1.output\n",
      "encoder.layer.1.output.dense\n",
      "encoder.layer.1.output.LayerNorm\n",
      "encoder.layer.1.output.dropout\n",
      "encoder.layer.2\n",
      "encoder.layer.2.attention\n",
      "encoder.layer.2.attention.self\n",
      "encoder.layer.2.attention.self.query\n",
      "encoder.layer.2.attention.self.key\n",
      "encoder.layer.2.attention.self.value\n",
      "encoder.layer.2.attention.self.dropout\n",
      "encoder.layer.2.attention.output\n",
      "encoder.layer.2.attention.output.dense\n",
      "encoder.layer.2.attention.output.LayerNorm\n",
      "encoder.layer.2.attention.output.dropout\n",
      "encoder.layer.2.intermediate\n",
      "encoder.layer.2.intermediate.dense\n",
      "encoder.layer.2.intermediate.intermediate_act_fn\n",
      "encoder.layer.2.output\n",
      "encoder.layer.2.output.dense\n",
      "encoder.layer.2.output.LayerNorm\n",
      "encoder.layer.2.output.dropout\n",
      "encoder.layer.3\n",
      "encoder.layer.3.attention\n",
      "encoder.layer.3.attention.self\n",
      "encoder.layer.3.attention.self.query\n",
      "encoder.layer.3.attention.self.key\n",
      "encoder.layer.3.attention.self.value\n",
      "encoder.layer.3.attention.self.dropout\n",
      "encoder.layer.3.attention.output\n",
      "encoder.layer.3.attention.output.dense\n",
      "encoder.layer.3.attention.output.LayerNorm\n",
      "encoder.layer.3.attention.output.dropout\n",
      "encoder.layer.3.intermediate\n",
      "encoder.layer.3.intermediate.dense\n",
      "encoder.layer.3.intermediate.intermediate_act_fn\n",
      "encoder.layer.3.output\n",
      "encoder.layer.3.output.dense\n",
      "encoder.layer.3.output.LayerNorm\n",
      "encoder.layer.3.output.dropout\n",
      "encoder.layer.4\n",
      "encoder.layer.4.attention\n",
      "encoder.layer.4.attention.self\n",
      "encoder.layer.4.attention.self.query\n",
      "encoder.layer.4.attention.self.key\n",
      "encoder.layer.4.attention.self.value\n",
      "encoder.layer.4.attention.self.dropout\n",
      "encoder.layer.4.attention.output\n",
      "encoder.layer.4.attention.output.dense\n",
      "encoder.layer.4.attention.output.LayerNorm\n",
      "encoder.layer.4.attention.output.dropout\n",
      "encoder.layer.4.intermediate\n",
      "encoder.layer.4.intermediate.dense\n",
      "encoder.layer.4.intermediate.intermediate_act_fn\n",
      "encoder.layer.4.output\n",
      "encoder.layer.4.output.dense\n",
      "encoder.layer.4.output.LayerNorm\n",
      "encoder.layer.4.output.dropout\n",
      "encoder.layer.5\n",
      "encoder.layer.5.attention\n",
      "encoder.layer.5.attention.self\n",
      "encoder.layer.5.attention.self.query\n",
      "encoder.layer.5.attention.self.key\n",
      "encoder.layer.5.attention.self.value\n",
      "encoder.layer.5.attention.self.dropout\n",
      "encoder.layer.5.attention.output\n",
      "encoder.layer.5.attention.output.dense\n",
      "encoder.layer.5.attention.output.LayerNorm\n",
      "encoder.layer.5.attention.output.dropout\n",
      "encoder.layer.5.intermediate\n",
      "encoder.layer.5.intermediate.dense\n",
      "encoder.layer.5.intermediate.intermediate_act_fn\n",
      "encoder.layer.5.output\n",
      "encoder.layer.5.output.dense\n",
      "encoder.layer.5.output.LayerNorm\n",
      "encoder.layer.5.output.dropout\n",
      "pooler\n",
      "pooler.dense\n",
      "pooler.activation\n"
     ]
    }
   ],
   "source": [
    "for name, layer in model_student.named_modules():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "839646ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:10.430740Z",
     "iopub.status.busy": "2023-04-15T11:28:10.430230Z",
     "iopub.status.idle": "2023-04-15T11:28:10.440534Z",
     "shell.execute_reply": "2023-04-15T11:28:10.438885Z"
    },
    "papermill": {
     "duration": 0.023194,
     "end_time": "2023-04-15T11:28:10.443384",
     "exception": false,
     "start_time": "2023-04-15T11:28:10.420190",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model size: 313.265MB\n"
     ]
    }
   ],
   "source": [
    "param_size = 0\n",
    "for param in model_student.parameters():\n",
    "    param_size += param.nelement() * param.element_size()\n",
    "buffer_size = 0\n",
    "for buffer in model.buffers():\n",
    "    buffer_size += buffer.nelement() * buffer.element_size()\n",
    "\n",
    "size_all_mb = (param_size + buffer_size) / 1024**2\n",
    "print('model size: {:.3f}MB'.format(size_all_mb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c5439dbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:10.461115Z",
     "iopub.status.busy": "2023-04-15T11:28:10.459477Z",
     "iopub.status.idle": "2023-04-15T11:28:10.524108Z",
     "shell.execute_reply": "2023-04-15T11:28:10.522554Z"
    },
    "papermill": {
     "duration": 0.076486,
     "end_time": "2023-04-15T11:28:10.527038",
     "exception": false,
     "start_time": "2023-04-15T11:28:10.450552",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_student.load_state_dict(compressed_sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e6b50331",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:10.545146Z",
     "iopub.status.busy": "2023-04-15T11:28:10.544190Z",
     "iopub.status.idle": "2023-04-15T11:28:11.074214Z",
     "shell.execute_reply": "2023-04-15T11:28:11.072618Z"
    },
    "papermill": {
     "duration": 0.544023,
     "end_time": "2023-04-15T11:28:11.078321",
     "exception": false,
     "start_time": "2023-04-15T11:28:10.534298",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_student.save_pretrained('roberta_base_6layers_student', from_pt=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2a5ae2d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:11.095586Z",
     "iopub.status.busy": "2023-04-15T11:28:11.095137Z",
     "iopub.status.idle": "2023-04-15T11:28:13.286171Z",
     "shell.execute_reply": "2023-04-15T11:28:13.284914Z"
    },
    "papermill": {
     "duration": 2.203542,
     "end_time": "2023-04-15T11:28:13.289177",
     "exception": false,
     "start_time": "2023-04-15T11:28:11.085635",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at ../working/roberta_base_6layers_student and are newly initialized: ['lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_check = RobertaForMaskedLM.from_pretrained('../working/roberta_base_6layers_student',config = config )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9a0907e8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-15T11:28:13.306301Z",
     "iopub.status.busy": "2023-04-15T11:28:13.305501Z",
     "iopub.status.idle": "2023-04-15T11:28:14.443102Z",
     "shell.execute_reply": "2023-04-15T11:28:14.441131Z"
    },
    "papermill": {
     "duration": 1.150254,
     "end_time": "2023-04-15T11:28:14.446562",
     "exception": false,
     "start_time": "2023-04-15T11:28:13.296308",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config.json  pytorch_model.bin\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../working/roberta_base_6layers_student"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52c33aae",
   "metadata": {
    "papermill": {
     "duration": 0.007666,
     "end_time": "2023-04-15T11:28:14.461506",
     "exception": false,
     "start_time": "2023-04-15T11:28:14.453840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 41.848038,
   "end_time": "2023-04-15T11:28:16.501338",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-04-15T11:27:34.653300",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "09b13ec38291487792da4b6c1797772e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "28257ca36342473f826322aef224d6b1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "357d1f3d15eb4b078d3cc6f3c468ae9d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "36f84da189e344ee910b66c054c423a3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5810ca30a75c4b3c90a550762b606687": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_36f84da189e344ee910b66c054c423a3",
       "max": 481.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6d46551b3eb942a7a1431e7f559a2c3b",
       "value": 481.0
      }
     },
     "655a55d1c92343faa911b203e9f98efb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_09b13ec38291487792da4b6c1797772e",
       "placeholder": "​",
       "style": "IPY_MODEL_28257ca36342473f826322aef224d6b1",
       "value": "Downloading (…)lve/main/config.json: 100%"
      }
     },
     "6d46551b3eb942a7a1431e7f559a2c3b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "95019f845056445c84c36440e1ba4b36": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "96a22f33e7e84ff6acfb5b27e135ae3a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_95019f845056445c84c36440e1ba4b36",
       "placeholder": "​",
       "style": "IPY_MODEL_ac1d75d6a1cc412987e21fcc0c292125",
       "value": " 481/481 [00:00&lt;00:00, 18.5kB/s]"
      }
     },
     "a1b7f9ab3e594f109ab13a217f258584": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_655a55d1c92343faa911b203e9f98efb",
        "IPY_MODEL_5810ca30a75c4b3c90a550762b606687",
        "IPY_MODEL_96a22f33e7e84ff6acfb5b27e135ae3a"
       ],
       "layout": "IPY_MODEL_357d1f3d15eb4b078d3cc6f3c468ae9d"
      }
     },
     "ac1d75d6a1cc412987e21fcc0c292125": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}