llm-finetuning

1
{
2
  "cells": [
3
    {
4
      "cell_type": "markdown",
5
      "metadata": {
6
        "id": "view-in-github",
7
        "colab_type": "text"
8
      },
9
      "source": [
10
        "<a href=\"https://colab.research.google.com/github/ashishpatel26/LLM-Finetuning/blob/main/12_Fine_tuning_Microsoft_Phi_1_5b_on_custom_dataset(dialogstudio).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
11
      ]
12
    },
13
    {
14
      "cell_type": "code",
15
      "execution_count": 1,
16
      "metadata": {
17
        "colab": {
18
          "base_uri": "https://localhost:8080/"
19
        },
20
        "id": "SwcwXRajHelL",
21
        "outputId": "df5a4f77-5b26-42e9-b04d-7d238c321c41"
22
      },
23
      "outputs": [
24
        {
25
          "output_type": "stream",
26
          "name": "stdout",
27
          "text": [
28
            "Thu Sep 21 11:23:52 2023       \n",
29
            "+-----------------------------------------------------------------------------+\n",
30
            "| NVIDIA-SMI 525.105.17   Driver Version: 525.105.17   CUDA Version: 12.0     |\n",
31
            "|-------------------------------+----------------------+----------------------+\n",
32
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
33
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
34
            "|                               |                      |               MIG M. |\n",
35
            "|===============================+======================+======================|\n",
36
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
37
            "| N/A   60C    P8    11W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
38
            "|                               |                      |                  N/A |\n",
39
            "+-------------------------------+----------------------+----------------------+\n",
40
            "                                                                               \n",
41
            "+-----------------------------------------------------------------------------+\n",
42
            "| Processes:                                                                  |\n",
43
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
44
            "|        ID   ID                                                   Usage      |\n",
45
            "|=============================================================================|\n",
46
            "|  No running processes found                                                 |\n",
47
            "+-----------------------------------------------------------------------------+\n"
48
          ]
49
        }
50
      ],
51
      "source": [
52
        "!nvidia-smi"
53
      ]
54
    },
55
    {
56
      "cell_type": "code",
57
      "source": [
58
        "!pip install accelerate transformers einops datasets peft bitsandbytes trl"
59
      ],
60
      "metadata": {
61
        "colab": {
62
          "base_uri": "https://localhost:8080/"
63
        },
64
        "id": "wH8jsePrSGTH",
65
        "outputId": "53bf1128-bbbb-4319-c7bb-21820389d3f6"
66
      },
67
      "execution_count": 2,
68
      "outputs": [
69
        {
70
          "output_type": "stream",
71
          "name": "stdout",
72
          "text": [
73
            "Collecting accelerate\n",
74
            "  Downloading accelerate-0.23.0-py3-none-any.whl (258 kB)\n",
75
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m258.1/258.1 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
76
            "\u001b[?25hCollecting transformers\n",
77
            "  Downloading transformers-4.33.2-py3-none-any.whl (7.6 MB)\n",
78
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
79
            "\u001b[?25hCollecting einops\n",
80
            "  Downloading einops-0.6.1-py3-none-any.whl (42 kB)\n",
81
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
82
            "\u001b[?25hCollecting datasets\n",
83
            "  Downloading datasets-2.14.5-py3-none-any.whl (519 kB)\n",
84
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.6/519.6 kB\u001b[0m \u001b[31m26.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
85
            "\u001b[?25hCollecting peft\n",
86
            "  Downloading peft-0.5.0-py3-none-any.whl (85 kB)\n",
87
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.6/85.6 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
88
            "\u001b[?25hCollecting bitsandbytes\n",
89
            "  Downloading bitsandbytes-0.41.1-py3-none-any.whl (92.6 MB)\n",
90
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
91
            "\u001b[?25hCollecting trl\n",
92
            "  Downloading trl-0.7.1-py3-none-any.whl (117 kB)\n",
93
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.0/118.0 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
94
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
95
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.1)\n",
96
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
97
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
98
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
99
            "Collecting huggingface-hub (from accelerate)\n",
100
            "  Downloading huggingface_hub-0.17.2-py3-none-any.whl (294 kB)\n",
101
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m294.9/294.9 kB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
102
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
103
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
104
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
105
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
106
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
107
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m114.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
108
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
109
            "  Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
110
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m81.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
111
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
112
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
113
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
114
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
115
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
116
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
117
            "Collecting xxhash (from datasets)\n",
118
            "  Downloading xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
119
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
120
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
121
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
122
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
123
            "\u001b[?25hRequirement already satisfied: fsspec[http]<2023.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
124
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.5)\n",
125
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
126
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.2.0)\n",
127
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
128
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
129
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
130
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
131
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
132
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.5.0)\n",
133
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
134
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.4)\n",
135
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
136
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
137
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
138
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
139
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
140
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.4.1)\n",
141
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n",
142
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
143
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
144
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
145
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
146
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
147
            "Installing collected packages: tokenizers, safetensors, bitsandbytes, xxhash, einops, dill, multiprocess, huggingface-hub, transformers, datasets, accelerate, trl, peft\n",
148
            "Successfully installed accelerate-0.23.0 bitsandbytes-0.41.1 datasets-2.14.5 dill-0.3.7 einops-0.6.1 huggingface-hub-0.17.2 multiprocess-0.70.15 peft-0.5.0 safetensors-0.3.3 tokenizers-0.13.3 transformers-4.33.2 trl-0.7.1 xxhash-3.3.0\n"
149
          ]
150
        }
151
      ]
152
    },
153
    {
154
      "cell_type": "code",
155
      "execution_count": 3,
156
      "metadata": {
157
        "id": "rpf1Z0k4RJM6"
158
      },
159
      "outputs": [],
160
      "source": [
161
        "# !pip install -Uqqq pip --progress-bar off\n",
162
        "# !pip install -qqq torch==2.0.1 --progress-bar off\n",
163
        "# !pip install -qqq transformers==4.32.1 --progress-bar off\n",
164
        "# !pip install -qqq datasets==2.14.4 --progress-bar off\n",
165
        "# !pip install -qqq peft==0.5.0 --progress-bar off\n",
166
        "# !pip install -qqq bitsandbytes==0.41.1 --progress-bar off\n",
167
        "# !pip install -qqq trl==0.7.1 --progress-bar off"
168
      ]
169
    },
170
    {
171
      "cell_type": "code",
172
      "execution_count": 6,
173
      "metadata": {
174
        "id": "P1PG0WSvRqVq"
175
      },
176
      "outputs": [],
177
      "source": [
178
        "import json\n",
179
        "import re\n",
180
        "from pprint import pprint\n",
181
        "import os\n",
182
        "\n",
183
        "import pandas as pd\n",
184
        "import torch\n",
185
        "from datasets import Dataset, load_dataset\n",
186
        "from huggingface_hub import notebook_login\n",
187
        "from peft import LoraConfig, PeftModel, get_peft_model\n",
188
        "from transformers import (\n",
189
        "    AutoModelForCausalLM,\n",
190
        "    AutoTokenizer,\n",
191
        "    BitsAndBytesConfig,\n",
192
        "    TrainingArguments,\n",
193
        "    DataCollatorForLanguageModeling,\n",
194
        "    BitsAndBytesConfig\n",
195
        ")\n",
196
        "from trl import SFTTrainer\n",
197
        "\n",
198
        "DEVICE = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
199
        "MODEL_NAME = \"microsoft/phi-1_5\""
200
      ]
201
    },
202
    {
203
      "cell_type": "markdown",
204
      "metadata": {
205
        "id": "4ixsX2Y4doEf"
206
      },
207
      "source": [
208
        "## Data"
209
      ]
210
    },
211
    {
212
      "cell_type": "code",
213
      "source": [
214
        "dataset = load_dataset(\"Salesforce/dialogstudio\", \"TweetSumm\")\n",
215
        "dataset"
216
      ],
217
      "metadata": {
218
        "colab": {
219
          "base_uri": "https://localhost:8080/"
220
        },
221
        "id": "Kc0CVTtUkWvl",
222
        "outputId": "b078c12b-4a06-49e1-f101-3fdc40a7f5dd"
223
      },
224
      "execution_count": 7,
225
      "outputs": [
226
        {
227
          "output_type": "execute_result",
228
          "data": {
229
            "text/plain": [
230
              "DatasetDict({\n",
231
              "    train: Dataset({\n",
232
              "        features: ['original dialog id', 'new dialog id', 'dialog index', 'original dialog info', 'log', 'prompt'],\n",
233
              "        num_rows: 879\n",
234
              "    })\n",
235
              "    validation: Dataset({\n",
236
              "        features: ['original dialog id', 'new dialog id', 'dialog index', 'original dialog info', 'log', 'prompt'],\n",
237
              "        num_rows: 110\n",
238
              "    })\n",
239
              "    test: Dataset({\n",
240
              "        features: ['original dialog id', 'new dialog id', 'dialog index', 'original dialog info', 'log', 'prompt'],\n",
241
              "        num_rows: 110\n",
242
              "    })\n",
243
              "})"
244
            ]
245
          },
246
          "metadata": {},
247
          "execution_count": 7
248
        }
249
      ]
250
    },
251
    {
252
      "cell_type": "code",
253
      "source": [
254
        "DEFAULT_SYSTEM_PROMPT = \"\"\"\n",
255
        "Below is a conversation between a human and an AI agent. Write a summary of the conversation.\n",
256
        "\"\"\".strip()\n",
257
        "\n",
258
        "\n",
259
        "def generate_training_prompt(\n",
260
        "    conversation: str, summary: str, system_prompt: str = DEFAULT_SYSTEM_PROMPT\n",
261
        ") -> str:\n",
262
        "    return f\"\"\"### Instruction: {system_prompt}\n",
263
        "\n",
264
        "### Input:\n",
265
        "{conversation.strip()}\n",
266
        "\n",
267
        "### Response:\n",
268
        "{summary}\n",
269
        "\"\"\".strip()"
270
      ],
271
      "metadata": {
272
        "id": "fbx71jhaMGK5"
273
      },
274
      "execution_count": 8,
275
      "outputs": []
276
    },
277
    {
278
      "cell_type": "code",
279
      "source": [
280
        "def clean_text(text):\n",
281
        "    text = re.sub(r\"http\\S+\", \"\", text)\n",
282
        "    text = re.sub(r\"@[^\\s]+\", \"\", text)\n",
283
        "    text = re.sub(r\"\\s+\", \" \", text)\n",
284
        "    return re.sub(r\"\\^[^ ]+\", \"\", text)\n",
285
        "\n",
286
        "\n",
287
        "def create_conversation_text(data_point):\n",
288
        "    text = \"\"\n",
289
        "    for item in data_point[\"log\"]:\n",
290
        "        user = clean_text(item[\"user utterance\"])\n",
291
        "        text += f\"user: {user.strip()}\\n\"\n",
292
        "\n",
293
        "        agent = clean_text(item[\"system response\"])\n",
294
        "        text += f\"agent: {agent.strip()}\\n\"\n",
295
        "\n",
296
        "    return text"
297
      ],
298
      "metadata": {
299
        "id": "gMfzUJVOR9Lr"
300
      },
301
      "execution_count": 9,
302
      "outputs": []
303
    },
304
    {
305
      "cell_type": "code",
306
      "source": [
307
        "def generate_text(data_point):\n",
308
        "    summaries = json.loads(data_point[\"original dialog info\"])[\"summaries\"][\n",
309
        "        \"abstractive_summaries\"\n",
310
        "    ]\n",
311
        "    summary = summaries[0]\n",
312
        "    summary = \" \".join(summary)\n",
313
        "\n",
314
        "    conversation_text = create_conversation_text(data_point)\n",
315
        "    return {\n",
316
        "        \"conversation\": conversation_text,\n",
317
        "        \"summary\": summary,\n",
318
        "        \"text\": generate_training_prompt(conversation_text, summary),\n",
319
        "    }"
320
      ],
321
      "metadata": {
322
        "id": "eRbskn48QNfW"
323
      },
324
      "execution_count": 10,
325
      "outputs": []
326
    },
327
    {
328
      "cell_type": "code",
329
      "source": [
330
        "example = generate_text(dataset[\"train\"][0])"
331
      ],
332
      "metadata": {
333
        "id": "J9VuGHGYSR5q"
334
      },
335
      "execution_count": 11,
336
      "outputs": []
337
    },
338
    {
339
      "cell_type": "code",
340
      "source": [
341
        "print(example[\"summary\"])"
342
      ],
343
      "metadata": {
344
        "colab": {
345
          "base_uri": "https://localhost:8080/"
346
        },
347
        "id": "ycfL7LvHSu2h",
348
        "outputId": "0ed47942-4843-4427-de4d-92f37c84504e"
349
      },
350
      "execution_count": 12,
351
      "outputs": [
352
        {
353
          "output_type": "stream",
354
          "name": "stdout",
355
          "text": [
356
            "Customer enquired about his Iphone and Apple watch which is not showing his any steps/activity and health activities. Agent is asking to move to DM and look into it.\n"
357
          ]
358
        }
359
      ]
360
    },
361
    {
362
      "cell_type": "code",
363
      "source": [
364
        "print(example[\"conversation\"])"
365
      ],
366
      "metadata": {
367
        "colab": {
368
          "base_uri": "https://localhost:8080/"
369
        },
370
        "id": "eL7EDMq_SxlJ",
371
        "outputId": "946ec8e8-58a5-4afb-abfa-49a9239097d0"
372
      },
373
      "execution_count": 13,
374
      "outputs": [
375
        {
376
          "output_type": "stream",
377
          "name": "stdout",
378
          "text": [
379
            "user: So neither my iPhone nor my Apple Watch are recording my steps/activity, and Health doesn’t recognise either source anymore for some reason. Any ideas? please read the above.\n",
380
            "agent: Let’s investigate this together. To start, can you tell us the software versions your iPhone and Apple Watch are running currently?\n",
381
            "user: My iPhone is on 11.1.2, and my watch is on 4.1.\n",
382
            "agent: Thank you. Have you tried restarting both devices since this started happening?\n",
383
            "user: I’ve restarted both, also un-paired then re-paired the watch.\n",
384
            "agent: Got it. When did you first notice that the two devices were not talking to each other. Do the two devices communicate through other apps such as Messages?\n",
385
            "user: Yes, everything seems fine, it’s just Health and activity.\n",
386
            "agent: Let’s move to DM and look into this a bit more. When reaching out in DM, let us know when this first started happening please. For example, did it start after an update or after installing a certain app?\n",
387
            "\n"
388
          ]
389
        }
390
      ]
391
    },
392
    {
393
      "cell_type": "code",
394
      "source": [
395
        "print(example[\"text\"])"
396
      ],
397
      "metadata": {
398
        "colab": {
399
          "base_uri": "https://localhost:8080/"
400
        },
401
        "id": "BbiL_o5dZZhg",
402
        "outputId": "af672c10-5160-4167-ad33-aeeea5b1ebbd"
403
      },
404
      "execution_count": 14,
405
      "outputs": [
406
        {
407
          "output_type": "stream",
408
          "name": "stdout",
409
          "text": [
410
            "### Instruction: Below is a conversation between a human and an AI agent. Write a summary of the conversation.\n",
411
            "\n",
412
            "### Input:\n",
413
            "user: So neither my iPhone nor my Apple Watch are recording my steps/activity, and Health doesn’t recognise either source anymore for some reason. Any ideas? please read the above.\n",
414
            "agent: Let’s investigate this together. To start, can you tell us the software versions your iPhone and Apple Watch are running currently?\n",
415
            "user: My iPhone is on 11.1.2, and my watch is on 4.1.\n",
416
            "agent: Thank you. Have you tried restarting both devices since this started happening?\n",
417
            "user: I’ve restarted both, also un-paired then re-paired the watch.\n",
418
            "agent: Got it. When did you first notice that the two devices were not talking to each other. Do the two devices communicate through other apps such as Messages?\n",
419
            "user: Yes, everything seems fine, it’s just Health and activity.\n",
420
            "agent: Let’s move to DM and look into this a bit more. When reaching out in DM, let us know when this first started happening please. For example, did it start after an update or after installing a certain app?\n",
421
            "\n",
422
            "### Response:\n",
423
            "Customer enquired about his Iphone and Apple watch which is not showing his any steps/activity and health activities. Agent is asking to move to DM and look into it.\n"
424
          ]
425
        }
426
      ]
427
    },
428
    {
429
      "cell_type": "code",
430
      "source": [
431
        "def process_dataset(data: Dataset):\n",
432
        "    return (\n",
433
        "        data.shuffle(seed=42)\n",
434
        "        .map(generate_text)\n",
435
        "        .remove_columns(\n",
436
        "            [\n",
437
        "                \"original dialog id\",\n",
438
        "                \"new dialog id\",\n",
439
        "                \"dialog index\",\n",
440
        "                \"original dialog info\",\n",
441
        "                \"log\",\n",
442
        "                \"prompt\",\n",
443
        "            ]\n",
444
        "        )\n",
445
        "    )"
446
      ],
447
      "metadata": {
448
        "id": "jKidKeUpZkPb"
449
      },
450
      "execution_count": 15,
451
      "outputs": []
452
    },
453
    {
454
      "cell_type": "code",
455
      "source": [
456
        "dataset[\"train\"] = process_dataset(dataset[\"train\"])\n",
457
        "dataset[\"validation\"] = process_dataset(dataset[\"validation\"])\n",
458
        "dataset[\"test\"] = process_dataset(dataset[\"test\"])"
459
      ],
460
      "metadata": {
461
        "id": "XHy1pVjlaLtm"
462
      },
463
      "execution_count": 16,
464
      "outputs": []
465
    },
466
    {
467
      "cell_type": "markdown",
468
      "metadata": {
469
        "id": "usDeQuT2Wssl"
470
      },
471
      "source": [
472
        "## Model"
473
      ]
474
    },
475
    {
476
      "cell_type": "code",
477
      "execution_count": 24,
478
      "metadata": {
479
        "colab": {
480
          "base_uri": "https://localhost:8080/",
481
          "height": 145,
482
          "referenced_widgets": [
483
            "96ef5be98b5c4f2c9945b1a8c0989d7b",
484
            "29aca8e4365b4bee99d5292d8e74cdd6",
485
            "c83294f6ba864858a05d5222d84eb0c3",
486
            "7254587396ba4701835ee666ee3b697b",
487
            "896def48ce8148f18a13b0a46d0919b3",
488
            "5bce118ef58b43b6ab295e6272d84668",
489
            "a68cf26cf964423b8ce413bc6c8c5299",
490
            "ed2f736177914d2abb1c2efb6ecce6bb",
491
            "6c2ff22a40b543b6ac26ccd2b7298a61",
492
            "bb5fc1bf3f4c465e8b0846f22e71a23b",
493
            "42cae2119e5e49b9b7250a6f96f373bc",
494
            "8701e3ece5b04773a157dacc8f793374",
495
            "b3b75ce135e049bbacd598eac3d579ce",
496
            "6bb60bf24297418384b709e106c6c71a",
497
            "4b209751747c4ff98d905702b4ba3ab6",
498
            "ca3fcc5229864c2d8030b8e1bce3680a",
499
            "e43a98b2e5ba41a5a483c9320f982960",
500
            "f18eecc689234406a32b478a88379011",
501
            "dc6a98f4481a4d158ffd66d036d9d391",
502
            "c3e8d37c64af4b1eaed0569fd32c3841",
503
            "2cf7de58a08f4a10a0388fd5284adc61",
504
            "3e55c582114c43d19f9a88819739d9a5",
505
            "cdd259b528bf4cd7b1c2f834cec0372d",
506
            "1440bf6528a844f399aea3997b0348ba",
507
            "60af265316074adab78687ac53070910",
508
            "1c9d8afeb2644c65909f6e689e1af0af",
509
            "06d7ce85d9784f58add0583feda74530",
510
            "bcacfb2e6c464423b5e8bc0aef675fdd",
511
            "88c5cc8d66d8408cbe2bfedb365723f3",
512
            "814bac58a4974f70b17b5e19f88b042f",
513
            "3502bced8abc4cfc87300881857c4e08",
514
            "c726307cdc3c4598ad45879f3f29e60c"
515
          ]
516
        },
517
        "id": "3cLg8cYCdv74",
518
        "outputId": "b602a1eb-af6c-4069-a8f0-ee389e86ea38"
519
      },
520
      "outputs": [
521
        {
522
          "output_type": "display_data",
523
          "data": {
524
            "text/plain": [
525
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
526
            ],
527
            "application/vnd.jupyter.widget-view+json": {
528
              "version_major": 2,
529
              "version_minor": 0,
530
              "model_id": "96ef5be98b5c4f2c9945b1a8c0989d7b"
531
            }
532
          },
533
          "metadata": {}
534
        }
535
      ],
536
      "source": [
537
        "notebook_login()"
538
      ]
539
    },
540
    {
541
      "cell_type": "code",
542
      "execution_count": 16,
543
      "metadata": {
544
        "id": "GwxKEVnKWWss"
545
      },
546
      "outputs": [],
547
      "source": [
548
        "def create_model_and_tokenizer():\n",
549
        "    bnb_config = BitsAndBytesConfig(\n",
550
        "        load_in_4bit=True,\n",
551
        "        bnb_4bit_use_double_quant=True,\n",
552
        "        bnb_4bit_quant_type=\"nf4\",\n",
553
        "        bnb_4bit_compute_dtype=torch.float16,\n",
554
        "    )\n",
555
        "\n",
556
        "    model = AutoModelForCausalLM.from_pretrained(\n",
557
        "        MODEL_NAME,\n",
558
        "        device_map={\"\":0},\n",
559
        "        trust_remote_code=True,\n",
560
        "        quantization_config=bnb_config\n",
561
        "    )\n",
562
        "\n",
563
        "    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
564
        "    tokenizer.pad_token = tokenizer.eos_token\n",
565
        "\n",
566
        "    return model, tokenizer"
567
      ]
568
    },
569
    {
570
      "cell_type": "code",
571
      "execution_count": 17,
572
      "metadata": {
573
        "colab": {
574
          "base_uri": "https://localhost:8080/",
575
          "height": 473,
576
          "referenced_widgets": [
577
            "ee3793650a24458c8f1c9da6f6cf7930",
578
            "db88e2bb13fa425aaccc2f7a32f2a946",
579
            "9f1b94961d43444697b621f5f2a80392",
580
            "bfa4594314d6437994f2cb57311d97a6",
581
            "dd13edfc70174149b25a15b0858ba61c",
582
            "2e0685af66bb4e52b72b8e0159df3531",
583
            "0d86661bee2d497fa856892c6b6f3f46",
584
            "94916f236b0d4e8a8733871e84df1147",
585
            "97d6e28ba91e45cd899719bb5bc2b6ab",
586
            "118f1b7073b847ef987036cd82499d0b",
587
            "dd77325d1d8d417eb6544ae0cbc15133",
588
            "794c3466fdc34e549aaefbc106537e16",
589
            "cb4a5c788e9d44818ff61e2551ed750e",
590
            "fec4fa9c00d6429791bceeccc053bc05",
591
            "fef1d4090bcf4d77895d45567bb1ff41",
592
            "06a0a6459ff14187acac58481400bf05",
593
            "6caf5b09e0f44382807c41193a2963d3",
594
            "6c95ad4f7ae24185a3fa9dbf9f8f68a2",
595
            "69b2fdbe3a5e4682963987fb914afec5",
596
            "38e5964aa7a744439c7e5ad26c162ead",
597
            "8f1d717eba6942b8aca16cbc49bc6a6f",
598
            "0168b508956141269412e88990ee3d67",
599
            "e8b48f5b7e2b4d8d9407cb667092a3e8",
600
            "8b1317e6869e439092c1d880137f62fd",
601
            "72a199e2dc1b4d63984493e6bf1bdb8c",
602
            "2296ca45af0c4cec80d01fb7e33f278f",
603
            "3dd72eb426fa4e6e919276bb4e91f60f",
604
            "2d01ed4a650c4bcf9d07f020ae98081d",
605
            "51ac6641cd1441998abadc1993d4ef96",
606
            "520db9e6803d48fca60b1063e10cf260",
607
            "677f54e2735f407088724a38212c0171",
608
            "49fd03a8e4954f5caf410898882324c8",
609
            "63e68a09d85a4afd90d125bc61983d7a",
610
            "3672b47a27d64466995731f7475dff05",
611
            "27905804572740018fa7c00a6de8e82d",
612
            "dd8375f821bd4283b3f906aea5c23d07",
613
            "a9f84e0028e440828def8cee6ae317ee",
614
            "0c068bce19d74f7b835c90221a1cde38",
615
            "f9df8cdf043a4aaab82d55f24c17a3d8",
616
            "4efaf38f2eaa46799c2afecb55ccf14c",
617
            "498424778ff94f7ea6e6f32e8cd95868",
618
            "6b654ce3193745208af00288d6f9d5f9",
619
            "681653beca9b4a6591f7008e0ff5ac3b",
620
            "997bd19e08ff48bc92a83487598be156",
621
            "27e56e4591da430fb5f62b5d49df7e9e",
622
            "b90e6e929c164e3384cb0067555f66a4",
623
            "5fb49e2af10d452c9558f6f2b7551b16",
624
            "a6547c497b5b44b59c1de056965de351",
625
            "8732438e1da948e6a4f74cde7a25eb57",
626
            "cdd90757935f45668a81520522e3f057",
627
            "aa2269ddef7f4e66a42973e5fd3e568f",
628
            "a9d745f127214a4289bf880d65899dd3",
629
            "90884fd8a8d24859b4f94aac4041cfe4",
630
            "0fced73c513242fabd2fc4eb30df1e21",
631
            "22d1d55d6580458fb8fdb4af38ec4690",
632
            "a4acb6dc90144e729a615bb9b6e2db55",
633
            "3bc8f617a46e47fb8676ffee9cc25fba",
634
            "fe82a3b74c5b4f79aa367fe5d8f79345",
635
            "387d62eb068e44619908b006079eacd7",
636
            "e6d16b50198a450c876640e34d17140e",
637
            "b4a34fe788034852876cc1886c28042f",
638
            "265d6ad184a74be9b9e8d7378bb98f8d",
639
            "e25b91678ed246cd997cec87e918c079",
640
            "7bf39b13ce4148ca8fe508c73b63e413",
641
            "b0e8f79bf6b74f08801f385b4c7aadee",
642
            "8a8d6a4614e14aaeb03e4bfa9b8a0741",
643
            "f14e8afa14b74002af2be74b84d8b033",
644
            "17cc56f07af34e588c3d3e0cdba0b458",
645
            "9439477ef5b64c1ba5c517d76c2c460f",
646
            "fc8787cef48c4befae588efa56f99adc",
647
            "fd97aec74a1148b0b94de3f2194a0558",
648
            "cf54b45acb1d40d4a34189e76ffa831c",
649
            "1cbd0d65bcf9496c85e07225be76ffdf",
650
            "4689e3a07503492eb11986bf5e8a864f",
651
            "4d443929f61f4991bc88caeb94adaaa3",
652
            "9f60e630e47841ba91855613da83e295",
653
            "12332fa62a8f4b6f974eb37bf8eddaf2",
654
            "e333b53d363b47648ab6694e25b68ad7",
655
            "5e9ebdfb88fa413b88fdf2dd5bc2a3a1",
656
            "058ff0fedb9545199d11930a9aa1deec",
657
            "6996b093bc5a4ba385dfd134f687e8db",
658
            "e1a107dbb4784261b608c382895e7daf",
659
            "73ef312d625542ccaf7994e9300acb1b",
660
            "541a328e5cb84163a1295eeee4494d3f",
661
            "db228a2b64c148d98f5db9bfbf2ac5af",
662
            "85455342596f4e90b98cc41da60b8cb6",
663
            "22b767cd81db4953959e78d71f604dce",
664
            "9fb8334f33e84dc3a4bb3deaf1e7777c",
665
            "f6ee0db62fe14f1e86c972a4a157e315",
666
            "f57698a07edd4a43b0b1f55e0d6b4a02",
667
            "9adaf92c8acf4b7ea94e5591cfe27707",
668
            "f1d7c8a0a78041b184d64a69f7ed0789",
669
            "f4ca5f16039b4258b9c83768d211e757",
670
            "864288a708f545629900a29fb05f3d8b",
671
            "746c0bfc3d90446ca10a16d7838441aa",
672
            "889a67c857fe40c89d0d8e523a66ae08",
673
            "f66ce8552d5d4355b90e934a1c8c04cd",
674
            "0c3a38f76e6841a38f7a0e545c9c4211",
675
            "e75c5a29c59747dda108212a336c7e2c",
676
            "ce6b953129c945c7a7066d0f07385c92",
677
            "6c5a70913c6542be93085a3eefb2b1ce",
678
            "bd5a93537e9d49ec86233397c0ec2a6f",
679
            "36fad39ecf0347079291103b73c3f6dd",
680
            "14f4fc13863648599cef5b830ab1bd9f",
681
            "4212106d48a34279a45890b62879ec87",
682
            "2bee159b595643619607aa10b45c186f",
683
            "c2502539d4eb413da194fae615d6e6bf",
684
            "95ab78e1349d466d90df3390c0a6ceff",
685
            "9ca4b24b9dbb4b5a9af5897c46069c49",
686
            "5e5449f0eda34f29903b59ca07ac189f",
687
            "cae6be4ca33c48acb53cfcad9cbad985",
688
            "985bd9a745764ceab05a8e665eefe591",
689
            "548672800a5b4e5ca72df6d63e01f9d3",
690
            "62b898efd2ed4810bc8bc7de4e7feebf",
691
            "8de7a61d323e4914901a7cf62a916a07",
692
            "9ee95e5cee6d444588116346f6acedf2",
693
            "9baed57a94ed47e493fa23eb65c2526b",
694
            "1f3a418ebb47456ea041d7b5b3cf85d5",
695
            "fc0ff589460d47e5b97f745b748dd136",
696
            "b55ef28dff8b4b14b5b2cbf0438f8e44",
697
            "8be7a69bd51b448da2111a21822b7482"
698
          ]
699
        },
700
        "id": "qDVlGrUmdzdh",
701
        "outputId": "d068211c-3c4f-437e-855e-a4e721997466"
702
      },
703
      "outputs": [
704
        {
705
          "output_type": "display_data",
706
          "data": {
707
            "text/plain": [
708
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/880 [00:00<?, ?B/s]"
709
            ],
710
            "application/vnd.jupyter.widget-view+json": {
711
              "version_major": 2,
712
              "version_minor": 0,
713
              "model_id": "ee3793650a24458c8f1c9da6f6cf7930"
714
            }
715
          },
716
          "metadata": {}
717
        },
718
        {
719
          "output_type": "display_data",
720
          "data": {
721
            "text/plain": [
722
              "Downloading (…)former_sequential.py:   0%|          | 0.00/2.23k [00:00<?, ?B/s]"
723
            ],
724
            "application/vnd.jupyter.widget-view+json": {
725
              "version_major": 2,
726
              "version_minor": 0,
727
              "model_id": "794c3466fdc34e549aaefbc106537e16"
728
            }
729
          },
730
          "metadata": {}
731
        },
732
        {
733
          "output_type": "stream",
734
          "name": "stderr",
735
          "text": [
736
            "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-1_5:\n",
737
            "- configuration_mixformer_sequential.py\n",
738
            ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
739
          ]
740
        },
741
        {
742
          "output_type": "display_data",
743
          "data": {
744
            "text/plain": [
745
              "Downloading (…)former_sequential.py:   0%|          | 0.00/32.2k [00:00<?, ?B/s]"
746
            ],
747
            "application/vnd.jupyter.widget-view+json": {
748
              "version_major": 2,
749
              "version_minor": 0,
750
              "model_id": "e8b48f5b7e2b4d8d9407cb667092a3e8"
751
            }
752
          },
753
          "metadata": {}
754
        },
755
        {
756
          "output_type": "stream",
757
          "name": "stderr",
758
          "text": [
759
            "A new version of the following files was downloaded from https://huggingface.co/microsoft/phi-1_5:\n",
760
            "- modeling_mixformer_sequential.py\n",
761
            ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
762
          ]
763
        },
764
        {
765
          "output_type": "display_data",
766
          "data": {
767
            "text/plain": [
768
              "Downloading pytorch_model.bin:   0%|          | 0.00/2.84G [00:00<?, ?B/s]"
769
            ],
770
            "application/vnd.jupyter.widget-view+json": {
771
              "version_major": 2,
772
              "version_minor": 0,
773
              "model_id": "3672b47a27d64466995731f7475dff05"
774
            }
775
          },
776
          "metadata": {}
777
        },
778
        {
779
          "output_type": "display_data",
780
          "data": {
781
            "text/plain": [
782
              "Downloading (…)neration_config.json:   0%|          | 0.00/69.0 [00:00<?, ?B/s]"
783
            ],
784
            "application/vnd.jupyter.widget-view+json": {
785
              "version_major": 2,
786
              "version_minor": 0,
787
              "model_id": "27e56e4591da430fb5f62b5d49df7e9e"
788
            }
789
          },
790
          "metadata": {}
791
        },
792
        {
793
          "output_type": "display_data",
794
          "data": {
795
            "text/plain": [
796
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/237 [00:00<?, ?B/s]"
797
            ],
798
            "application/vnd.jupyter.widget-view+json": {
799
              "version_major": 2,
800
              "version_minor": 0,
801
              "model_id": "a4acb6dc90144e729a615bb9b6e2db55"
802
            }
803
          },
804
          "metadata": {}
805
        },
806
        {
807
          "output_type": "display_data",
808
          "data": {
809
            "text/plain": [
810
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/798k [00:00<?, ?B/s]"
811
            ],
812
            "application/vnd.jupyter.widget-view+json": {
813
              "version_major": 2,
814
              "version_minor": 0,
815
              "model_id": "f14e8afa14b74002af2be74b84d8b033"
816
            }
817
          },
818
          "metadata": {}
819
        },
820
        {
821
          "output_type": "display_data",
822
          "data": {
823
            "text/plain": [
824
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
825
            ],
826
            "application/vnd.jupyter.widget-view+json": {
827
              "version_major": 2,
828
              "version_minor": 0,
829
              "model_id": "e333b53d363b47648ab6694e25b68ad7"
830
            }
831
          },
832
          "metadata": {}
833
        },
834
        {
835
          "output_type": "display_data",
836
          "data": {
837
            "text/plain": [
838
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/2.11M [00:00<?, ?B/s]"
839
            ],
840
            "application/vnd.jupyter.widget-view+json": {
841
              "version_major": 2,
842
              "version_minor": 0,
843
              "model_id": "f6ee0db62fe14f1e86c972a4a157e315"
844
            }
845
          },
846
          "metadata": {}
847
        },
848
        {
849
          "output_type": "display_data",
850
          "data": {
851
            "text/plain": [
852
              "Downloading (…)in/added_tokens.json:   0%|          | 0.00/1.08k [00:00<?, ?B/s]"
853
            ],
854
            "application/vnd.jupyter.widget-view+json": {
855
              "version_major": 2,
856
              "version_minor": 0,
857
              "model_id": "ce6b953129c945c7a7066d0f07385c92"
858
            }
859
          },
860
          "metadata": {}
861
        },
862
        {
863
          "output_type": "display_data",
864
          "data": {
865
            "text/plain": [
866
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/99.0 [00:00<?, ?B/s]"
867
            ],
868
            "application/vnd.jupyter.widget-view+json": {
869
              "version_major": 2,
870
              "version_minor": 0,
871
              "model_id": "cae6be4ca33c48acb53cfcad9cbad985"
872
            }
873
          },
874
          "metadata": {}
875
        }
876
      ],
877
      "source": [
878
        "model, tokenizer = create_model_and_tokenizer()\n",
879
        "model.config.use_cache = False"
880
      ]
881
    },
882
    {
883
      "cell_type": "code",
884
      "execution_count": 18,
885
      "metadata": {
886
        "colab": {
887
          "base_uri": "https://localhost:8080/"
888
        },
889
        "id": "DRubXChbe7pL",
890
        "outputId": "41c5c6ca-e714-4cdb-fb33-42be895eb19d"
891
      },
892
      "outputs": [
893
        {
894
          "output_type": "execute_result",
895
          "data": {
896
            "text/plain": [
897
              "{'quant_method': <QuantizationMethod.BITS_AND_BYTES: 'bitsandbytes'>,\n",
898
              " 'load_in_8bit': False,\n",
899
              " 'load_in_4bit': True,\n",
900
              " 'llm_int8_threshold': 6.0,\n",
901
              " 'llm_int8_skip_modules': None,\n",
902
              " 'llm_int8_enable_fp32_cpu_offload': False,\n",
903
              " 'llm_int8_has_fp16_weight': False,\n",
904
              " 'bnb_4bit_quant_type': 'nf4',\n",
905
              " 'bnb_4bit_use_double_quant': True,\n",
906
              " 'bnb_4bit_compute_dtype': 'float16'}"
907
            ]
908
          },
909
          "metadata": {},
910
          "execution_count": 18
911
        }
912
      ],
913
      "source": [
914
        "model.config.quantization_config.to_dict()"
915
      ]
916
    },
917
    {
918
      "cell_type": "code",
919
      "execution_count": 19,
920
      "metadata": {
921
        "id": "1Sj65mDxNeJv"
922
      },
923
      "outputs": [],
924
      "source": [
925
        "peft_config = LoraConfig(\n",
926
        "    r=16,\n",
927
        "    lora_alpha=16,\n",
928
        "    target_modules=[\"Wqkv\", \"out_proj\"],\n",
929
        "    lora_dropout=0.05,\n",
930
        "    bias=\"none\",\n",
931
        "    task_type=\"CAUSAL_LM\"\n",
932
        ")"
933
      ]
934
    },
935
    {
936
      "cell_type": "code",
937
      "source": [
938
        "model = get_peft_model(model, peft_config)\n",
939
        "model.print_trainable_parameters()"
940
      ],
941
      "metadata": {
942
        "colab": {
943
          "base_uri": "https://localhost:8080/"
944
        },
945
        "id": "Vv11jEAJT9Pp",
946
        "outputId": "83cb3d0e-df99-4dad-f46a-cd19fe571130"
947
      },
948
      "execution_count": 20,
949
      "outputs": [
950
        {
951
          "output_type": "stream",
952
          "name": "stdout",
953
          "text": [
954
            "trainable params: 4,718,592 || all params: 1,422,989,312 || trainable%: 0.3315971497613047\n"
955
          ]
956
        }
957
      ]
958
    },
959
    {
960
      "cell_type": "markdown",
961
      "metadata": {
962
        "id": "vVSLIcQl8g9h"
963
      },
964
      "source": [
965
        "## Training"
966
      ]
967
    },
968
    {
969
      "cell_type": "code",
970
      "execution_count": 21,
971
      "metadata": {
972
        "colab": {
973
          "base_uri": "https://localhost:8080/",
974
          "height": 821
975
        },
976
        "id": "0FoXB0psS8hN",
977
        "outputId": "10d07019-3644-4b3f-8eff-230c7dba6af0"
978
      },
979
      "outputs": [
980
        {
981
          "output_type": "display_data",
982
          "data": {
983
            "text/plain": [
984
              "<IPython.core.display.Javascript object>"
985
            ],
986
            "application/javascript": [
987
              "\n",
988
              "        (async () => {\n",
989
              "            const url = new URL(await google.colab.kernel.proxyPort(6006, {'cache': true}));\n",
990
              "            url.searchParams.set('tensorboardColab', 'true');\n",
991
              "            const iframe = document.createElement('iframe');\n",
992
              "            iframe.src = url;\n",
993
              "            iframe.setAttribute('width', '100%');\n",
994
              "            iframe.setAttribute('height', '800');\n",
995
              "            iframe.setAttribute('frameborder', 0);\n",
996
              "            document.body.appendChild(iframe);\n",
997
              "        })();\n",
998
              "    "
999
            ]
1000
          },
1001
          "metadata": {}
1002
        }
1003
      ],
1004
      "source": [
1005
        "OUTPUT_DIR = \"experiments\"\n",
1006
        "\n",
1007
        "%load_ext tensorboard\n",
1008
        "%tensorboard --logdir experiments/runs"
1009
      ]
1010
    },
1011
    {
1012
      "cell_type": "code",
1013
      "execution_count": 22,
1014
      "metadata": {
1015
        "id": "gIoe9TqOof-V"
1016
      },
1017
      "outputs": [],
1018
      "source": [
1019
        "training_arguments = TrainingArguments(\n",
1020
        "        output_dir=\"phi-1_5-finetuned-dialogstudio\",\n",
1021
        "        per_device_train_batch_size=4,\n",
1022
        "        gradient_accumulation_steps=1,\n",
1023
        "        learning_rate=2e-4,\n",
1024
        "        lr_scheduler_type=\"cosine\",\n",
1025
        "        save_strategy=\"epoch\",\n",
1026
        "        logging_steps=1,\n",
1027
        "        max_steps=3,\n",
1028
        "        num_train_epochs=1,\n",
1029
        "        push_to_hub=True\n",
1030
        "    )"
1031
      ]
1032
    },
1033
    {
1034
      "cell_type": "code",
1035
      "execution_count": 25,
1036
      "metadata": {
1037
        "colab": {
1038
          "base_uri": "https://localhost:8080/"
1039
        },
1040
        "id": "ZL4Qa_cypL1f",
1041
        "outputId": "7d3c3821-8395-407c-8f95-1dbd1d479efb"
1042
      },
1043
      "outputs": [
1044
        {
1045
          "output_type": "stream",
1046
          "name": "stderr",
1047
          "text": [
1048
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:166: UserWarning: You didn't pass a `max_seq_length` argument to the SFTTrainer, this will default to 1024\n",
1049
            "  warnings.warn(\n"
1050
          ]
1051
        }
1052
      ],
1053
      "source": [
1054
        "trainer = SFTTrainer(\n",
1055
        "    model=model,\n",
1056
        "    train_dataset=dataset[\"train\"],\n",
1057
        "    eval_dataset=dataset[\"validation\"],\n",
1058
        "    peft_config=peft_config,\n",
1059
        "    dataset_text_field=\"text\",\n",
1060
        "    tokenizer=tokenizer,\n",
1061
        "    args=training_arguments,\n",
1062
        ")"
1063
      ]
1064
    },
1065
    {
1066
      "cell_type": "code",
1067
      "source": [
1068
        "trainer.train()"
1069
      ],
1070
      "metadata": {
1071
        "colab": {
1072
          "base_uri": "https://localhost:8080/",
1073
          "height": 241
1074
        },
1075
        "id": "y7LDwR6kJdkO",
1076
        "outputId": "e22ee88f-7c55-4251-8a3e-22bf75e5c777"
1077
      },
1078
      "execution_count": 26,
1079
      "outputs": [
1080
        {
1081
          "output_type": "stream",
1082
          "name": "stderr",
1083
          "text": [
1084
            "You're using a CodeGenTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
1085
          ]
1086
        },
1087
        {
1088
          "output_type": "display_data",
1089
          "data": {
1090
            "text/plain": [
1091
              "<IPython.core.display.HTML object>"
1092
            ],
1093
            "text/html": [
1094
              "\n",
1095
              "    <div>\n",
1096
              "      \n",
1097
              "      <progress value='3' max='3' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
1098
              "      [3/3 00:02, Epoch 0/1]\n",
1099
              "    </div>\n",
1100
              "    <table border=\"1\" class=\"dataframe\">\n",
1101
              "  <thead>\n",
1102
              " <tr style=\"text-align: left;\">\n",
1103
              "      <th>Step</th>\n",
1104
              "      <th>Training Loss</th>\n",
1105
              "    </tr>\n",
1106
              "  </thead>\n",
1107
              "  <tbody>\n",
1108
              "    <tr>\n",
1109
              "      <td>1</td>\n",
1110
              "      <td>3.080800</td>\n",
1111
              "    </tr>\n",
1112
              "    <tr>\n",
1113
              "      <td>2</td>\n",
1114
              "      <td>2.887100</td>\n",
1115
              "    </tr>\n",
1116
              "    <tr>\n",
1117
              "      <td>3</td>\n",
1118
              "      <td>3.302700</td>\n",
1119
              "    </tr>\n",
1120
              "  </tbody>\n",
1121
              "</table><p>"
1122
            ]
1123
          },
1124
          "metadata": {}
1125
        },
1126
        {
1127
          "output_type": "execute_result",
1128
          "data": {
1129
            "text/plain": [
1130
              "TrainOutput(global_step=3, training_loss=3.090197483698527, metrics={'train_runtime': 7.7717, 'train_samples_per_second': 1.544, 'train_steps_per_second': 0.386, 'total_flos': 22418657574912.0, 'train_loss': 3.090197483698527, 'epoch': 0.01})"
1131
            ]
1132
          },
1133
          "metadata": {},
1134
          "execution_count": 26
1135
        }
1136
      ]
1137
    },
1138
    {
1139
      "cell_type": "code",
1140
      "source": [
1141
        "trainer.evaluate()"
1142
      ],
1143
      "metadata": {
1144
        "colab": {
1145
          "base_uri": "https://localhost:8080/",
1146
          "height": 124
1147
        },
1148
        "id": "IRiQABpzeom6",
1149
        "outputId": "4e1c2f58-5f5f-4322-86a7-6df3a779223a"
1150
      },
1151
      "execution_count": 27,
1152
      "outputs": [
1153
        {
1154
          "output_type": "display_data",
1155
          "data": {
1156
            "text/plain": [
1157
              "<IPython.core.display.HTML object>"
1158
            ],
1159
            "text/html": [
1160
              "\n",
1161
              "    <div>\n",
1162
              "      \n",
1163
              "      <progress value='14' max='14' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
1164
              "      [14/14 00:11]\n",
1165
              "    </div>\n",
1166
              "    "
1167
            ]
1168
          },
1169
          "metadata": {}
1170
        },
1171
        {
1172
          "output_type": "execute_result",
1173
          "data": {
1174
            "text/plain": [
1175
              "{'eval_loss': 3.243016242980957,\n",
1176
              " 'eval_runtime': 12.0781,\n",
1177
              " 'eval_samples_per_second': 9.107,\n",
1178
              " 'eval_steps_per_second': 1.159,\n",
1179
              " 'epoch': 0.01}"
1180
            ]
1181
          },
1182
          "metadata": {},
1183
          "execution_count": 27
1184
        }
1185
      ]
1186
    },
1187
    {
1188
      "cell_type": "code",
1189
      "source": [
1190
        "trainer.push_to_hub()"
1191
      ],
1192
      "metadata": {
1193
        "colab": {
1194
          "base_uri": "https://localhost:8080/",
1195
          "height": 35
1196
        },
1197
        "id": "ZCBJa6mZaSv7",
1198
        "outputId": "7111feef-c4e3-4330-b291-437c19c41c33"
1199
      },
1200
      "execution_count": 28,
1201
      "outputs": [
1202
        {
1203
          "output_type": "execute_result",
1204
          "data": {
1205
            "text/plain": [
1206
              "'https://huggingface.co/ashishpatel26/phi-1_5-finetuned-dialogstudio/tree/main/'"
1207
            ],
1208
            "application/vnd.google.colaboratory.intrinsic+json": {
1209
              "type": "string"
1210
            }
1211
          },
1212
          "metadata": {},
1213
          "execution_count": 28
1214
        }
1215
      ]
1216
    },
1217
    {
1218
      "cell_type": "code",
1219
      "source": [
1220
        "trainer.save_model(\"phi-1_5-finetuned-dialogstudio\")"
1221
      ],
1222
      "metadata": {
1223
        "id": "jJdkDvxKOq8P"
1224
      },
1225
      "execution_count": 30,
1226
      "outputs": []
1227
    },
1228
    {
1229
      "cell_type": "code",
1230
      "execution_count": 30,
1231
      "metadata": {
1232
        "colab": {
1233
          "base_uri": "https://localhost:8080/"
1234
        },
1235
        "id": "WywDQRmGEVOJ",
1236
        "outputId": "acdd8465-32e1-4fc1-ad75-06831edf8d85"
1237
      },
1238
      "outputs": [
1239
        {
1240
          "output_type": "execute_result",
1241
          "data": {
1242
            "text/plain": [
1243
              "PeftModelForCausalLM(\n",
1244
              "  (base_model): LoraModel(\n",
1245
              "    (model): MixFormerSequentialForCausalLM(\n",
1246
              "      (layers): Sequential(\n",
1247
              "        (0): Embedding(\n",
1248
              "          (wte): Embedding(51200, 2048)\n",
1249
              "          (drop): Dropout(p=0.0, inplace=False)\n",
1250
              "        )\n",
1251
              "        (1): ParallelBlock(\n",
1252
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1253
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1254
              "          (mixer): MHA(\n",
1255
              "            (rotary_emb): RotaryEmbedding()\n",
1256
              "            (Wqkv): Linear4bit(\n",
1257
              "              in_features=2048, out_features=6144, bias=True\n",
1258
              "              (lora_dropout): ModuleDict(\n",
1259
              "                (default): Dropout(p=0.05, inplace=False)\n",
1260
              "              )\n",
1261
              "              (lora_A): ModuleDict(\n",
1262
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1263
              "              )\n",
1264
              "              (lora_B): ModuleDict(\n",
1265
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1266
              "              )\n",
1267
              "              (lora_embedding_A): ParameterDict()\n",
1268
              "              (lora_embedding_B): ParameterDict()\n",
1269
              "            )\n",
1270
              "            (out_proj): Linear4bit(\n",
1271
              "              in_features=2048, out_features=2048, bias=True\n",
1272
              "              (lora_dropout): ModuleDict(\n",
1273
              "                (default): Dropout(p=0.05, inplace=False)\n",
1274
              "              )\n",
1275
              "              (lora_A): ModuleDict(\n",
1276
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1277
              "              )\n",
1278
              "              (lora_B): ModuleDict(\n",
1279
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1280
              "              )\n",
1281
              "              (lora_embedding_A): ParameterDict()\n",
1282
              "              (lora_embedding_B): ParameterDict()\n",
1283
              "            )\n",
1284
              "            (inner_attn): SelfAttention(\n",
1285
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1286
              "            )\n",
1287
              "            (inner_cross_attn): CrossAttention(\n",
1288
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1289
              "            )\n",
1290
              "          )\n",
1291
              "          (mlp): MLP(\n",
1292
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1293
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1294
              "            (act): NewGELUActivation()\n",
1295
              "          )\n",
1296
              "        )\n",
1297
              "        (2): ParallelBlock(\n",
1298
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1299
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1300
              "          (mixer): MHA(\n",
1301
              "            (rotary_emb): RotaryEmbedding()\n",
1302
              "            (Wqkv): Linear4bit(\n",
1303
              "              in_features=2048, out_features=6144, bias=True\n",
1304
              "              (lora_dropout): ModuleDict(\n",
1305
              "                (default): Dropout(p=0.05, inplace=False)\n",
1306
              "              )\n",
1307
              "              (lora_A): ModuleDict(\n",
1308
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1309
              "              )\n",
1310
              "              (lora_B): ModuleDict(\n",
1311
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1312
              "              )\n",
1313
              "              (lora_embedding_A): ParameterDict()\n",
1314
              "              (lora_embedding_B): ParameterDict()\n",
1315
              "            )\n",
1316
              "            (out_proj): Linear4bit(\n",
1317
              "              in_features=2048, out_features=2048, bias=True\n",
1318
              "              (lora_dropout): ModuleDict(\n",
1319
              "                (default): Dropout(p=0.05, inplace=False)\n",
1320
              "              )\n",
1321
              "              (lora_A): ModuleDict(\n",
1322
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1323
              "              )\n",
1324
              "              (lora_B): ModuleDict(\n",
1325
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1326
              "              )\n",
1327
              "              (lora_embedding_A): ParameterDict()\n",
1328
              "              (lora_embedding_B): ParameterDict()\n",
1329
              "            )\n",
1330
              "            (inner_attn): SelfAttention(\n",
1331
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1332
              "            )\n",
1333
              "            (inner_cross_attn): CrossAttention(\n",
1334
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1335
              "            )\n",
1336
              "          )\n",
1337
              "          (mlp): MLP(\n",
1338
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1339
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1340
              "            (act): NewGELUActivation()\n",
1341
              "          )\n",
1342
              "        )\n",
1343
              "        (3): ParallelBlock(\n",
1344
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1345
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1346
              "          (mixer): MHA(\n",
1347
              "            (rotary_emb): RotaryEmbedding()\n",
1348
              "            (Wqkv): Linear4bit(\n",
1349
              "              in_features=2048, out_features=6144, bias=True\n",
1350
              "              (lora_dropout): ModuleDict(\n",
1351
              "                (default): Dropout(p=0.05, inplace=False)\n",
1352
              "              )\n",
1353
              "              (lora_A): ModuleDict(\n",
1354
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1355
              "              )\n",
1356
              "              (lora_B): ModuleDict(\n",
1357
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1358
              "              )\n",
1359
              "              (lora_embedding_A): ParameterDict()\n",
1360
              "              (lora_embedding_B): ParameterDict()\n",
1361
              "            )\n",
1362
              "            (out_proj): Linear4bit(\n",
1363
              "              in_features=2048, out_features=2048, bias=True\n",
1364
              "              (lora_dropout): ModuleDict(\n",
1365
              "                (default): Dropout(p=0.05, inplace=False)\n",
1366
              "              )\n",
1367
              "              (lora_A): ModuleDict(\n",
1368
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1369
              "              )\n",
1370
              "              (lora_B): ModuleDict(\n",
1371
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1372
              "              )\n",
1373
              "              (lora_embedding_A): ParameterDict()\n",
1374
              "              (lora_embedding_B): ParameterDict()\n",
1375
              "            )\n",
1376
              "            (inner_attn): SelfAttention(\n",
1377
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1378
              "            )\n",
1379
              "            (inner_cross_attn): CrossAttention(\n",
1380
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1381
              "            )\n",
1382
              "          )\n",
1383
              "          (mlp): MLP(\n",
1384
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1385
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1386
              "            (act): NewGELUActivation()\n",
1387
              "          )\n",
1388
              "        )\n",
1389
              "        (4): ParallelBlock(\n",
1390
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1391
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1392
              "          (mixer): MHA(\n",
1393
              "            (rotary_emb): RotaryEmbedding()\n",
1394
              "            (Wqkv): Linear4bit(\n",
1395
              "              in_features=2048, out_features=6144, bias=True\n",
1396
              "              (lora_dropout): ModuleDict(\n",
1397
              "                (default): Dropout(p=0.05, inplace=False)\n",
1398
              "              )\n",
1399
              "              (lora_A): ModuleDict(\n",
1400
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1401
              "              )\n",
1402
              "              (lora_B): ModuleDict(\n",
1403
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1404
              "              )\n",
1405
              "              (lora_embedding_A): ParameterDict()\n",
1406
              "              (lora_embedding_B): ParameterDict()\n",
1407
              "            )\n",
1408
              "            (out_proj): Linear4bit(\n",
1409
              "              in_features=2048, out_features=2048, bias=True\n",
1410
              "              (lora_dropout): ModuleDict(\n",
1411
              "                (default): Dropout(p=0.05, inplace=False)\n",
1412
              "              )\n",
1413
              "              (lora_A): ModuleDict(\n",
1414
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1415
              "              )\n",
1416
              "              (lora_B): ModuleDict(\n",
1417
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1418
              "              )\n",
1419
              "              (lora_embedding_A): ParameterDict()\n",
1420
              "              (lora_embedding_B): ParameterDict()\n",
1421
              "            )\n",
1422
              "            (inner_attn): SelfAttention(\n",
1423
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1424
              "            )\n",
1425
              "            (inner_cross_attn): CrossAttention(\n",
1426
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1427
              "            )\n",
1428
              "          )\n",
1429
              "          (mlp): MLP(\n",
1430
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1431
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1432
              "            (act): NewGELUActivation()\n",
1433
              "          )\n",
1434
              "        )\n",
1435
              "        (5): ParallelBlock(\n",
1436
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1437
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1438
              "          (mixer): MHA(\n",
1439
              "            (rotary_emb): RotaryEmbedding()\n",
1440
              "            (Wqkv): Linear4bit(\n",
1441
              "              in_features=2048, out_features=6144, bias=True\n",
1442
              "              (lora_dropout): ModuleDict(\n",
1443
              "                (default): Dropout(p=0.05, inplace=False)\n",
1444
              "              )\n",
1445
              "              (lora_A): ModuleDict(\n",
1446
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1447
              "              )\n",
1448
              "              (lora_B): ModuleDict(\n",
1449
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1450
              "              )\n",
1451
              "              (lora_embedding_A): ParameterDict()\n",
1452
              "              (lora_embedding_B): ParameterDict()\n",
1453
              "            )\n",
1454
              "            (out_proj): Linear4bit(\n",
1455
              "              in_features=2048, out_features=2048, bias=True\n",
1456
              "              (lora_dropout): ModuleDict(\n",
1457
              "                (default): Dropout(p=0.05, inplace=False)\n",
1458
              "              )\n",
1459
              "              (lora_A): ModuleDict(\n",
1460
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1461
              "              )\n",
1462
              "              (lora_B): ModuleDict(\n",
1463
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1464
              "              )\n",
1465
              "              (lora_embedding_A): ParameterDict()\n",
1466
              "              (lora_embedding_B): ParameterDict()\n",
1467
              "            )\n",
1468
              "            (inner_attn): SelfAttention(\n",
1469
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1470
              "            )\n",
1471
              "            (inner_cross_attn): CrossAttention(\n",
1472
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1473
              "            )\n",
1474
              "          )\n",
1475
              "          (mlp): MLP(\n",
1476
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1477
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1478
              "            (act): NewGELUActivation()\n",
1479
              "          )\n",
1480
              "        )\n",
1481
              "        (6): ParallelBlock(\n",
1482
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1483
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1484
              "          (mixer): MHA(\n",
1485
              "            (rotary_emb): RotaryEmbedding()\n",
1486
              "            (Wqkv): Linear4bit(\n",
1487
              "              in_features=2048, out_features=6144, bias=True\n",
1488
              "              (lora_dropout): ModuleDict(\n",
1489
              "                (default): Dropout(p=0.05, inplace=False)\n",
1490
              "              )\n",
1491
              "              (lora_A): ModuleDict(\n",
1492
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1493
              "              )\n",
1494
              "              (lora_B): ModuleDict(\n",
1495
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1496
              "              )\n",
1497
              "              (lora_embedding_A): ParameterDict()\n",
1498
              "              (lora_embedding_B): ParameterDict()\n",
1499
              "            )\n",
1500
              "            (out_proj): Linear4bit(\n",
1501
              "              in_features=2048, out_features=2048, bias=True\n",
1502
              "              (lora_dropout): ModuleDict(\n",
1503
              "                (default): Dropout(p=0.05, inplace=False)\n",
1504
              "              )\n",
1505
              "              (lora_A): ModuleDict(\n",
1506
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1507
              "              )\n",
1508
              "              (lora_B): ModuleDict(\n",
1509
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1510
              "              )\n",
1511
              "              (lora_embedding_A): ParameterDict()\n",
1512
              "              (lora_embedding_B): ParameterDict()\n",
1513
              "            )\n",
1514
              "            (inner_attn): SelfAttention(\n",
1515
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1516
              "            )\n",
1517
              "            (inner_cross_attn): CrossAttention(\n",
1518
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1519
              "            )\n",
1520
              "          )\n",
1521
              "          (mlp): MLP(\n",
1522
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1523
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1524
              "            (act): NewGELUActivation()\n",
1525
              "          )\n",
1526
              "        )\n",
1527
              "        (7): ParallelBlock(\n",
1528
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1529
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1530
              "          (mixer): MHA(\n",
1531
              "            (rotary_emb): RotaryEmbedding()\n",
1532
              "            (Wqkv): Linear4bit(\n",
1533
              "              in_features=2048, out_features=6144, bias=True\n",
1534
              "              (lora_dropout): ModuleDict(\n",
1535
              "                (default): Dropout(p=0.05, inplace=False)\n",
1536
              "              )\n",
1537
              "              (lora_A): ModuleDict(\n",
1538
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1539
              "              )\n",
1540
              "              (lora_B): ModuleDict(\n",
1541
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1542
              "              )\n",
1543
              "              (lora_embedding_A): ParameterDict()\n",
1544
              "              (lora_embedding_B): ParameterDict()\n",
1545
              "            )\n",
1546
              "            (out_proj): Linear4bit(\n",
1547
              "              in_features=2048, out_features=2048, bias=True\n",
1548
              "              (lora_dropout): ModuleDict(\n",
1549
              "                (default): Dropout(p=0.05, inplace=False)\n",
1550
              "              )\n",
1551
              "              (lora_A): ModuleDict(\n",
1552
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1553
              "              )\n",
1554
              "              (lora_B): ModuleDict(\n",
1555
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1556
              "              )\n",
1557
              "              (lora_embedding_A): ParameterDict()\n",
1558
              "              (lora_embedding_B): ParameterDict()\n",
1559
              "            )\n",
1560
              "            (inner_attn): SelfAttention(\n",
1561
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1562
              "            )\n",
1563
              "            (inner_cross_attn): CrossAttention(\n",
1564
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1565
              "            )\n",
1566
              "          )\n",
1567
              "          (mlp): MLP(\n",
1568
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1569
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1570
              "            (act): NewGELUActivation()\n",
1571
              "          )\n",
1572
              "        )\n",
1573
              "        (8): ParallelBlock(\n",
1574
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1575
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1576
              "          (mixer): MHA(\n",
1577
              "            (rotary_emb): RotaryEmbedding()\n",
1578
              "            (Wqkv): Linear4bit(\n",
1579
              "              in_features=2048, out_features=6144, bias=True\n",
1580
              "              (lora_dropout): ModuleDict(\n",
1581
              "                (default): Dropout(p=0.05, inplace=False)\n",
1582
              "              )\n",
1583
              "              (lora_A): ModuleDict(\n",
1584
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1585
              "              )\n",
1586
              "              (lora_B): ModuleDict(\n",
1587
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1588
              "              )\n",
1589
              "              (lora_embedding_A): ParameterDict()\n",
1590
              "              (lora_embedding_B): ParameterDict()\n",
1591
              "            )\n",
1592
              "            (out_proj): Linear4bit(\n",
1593
              "              in_features=2048, out_features=2048, bias=True\n",
1594
              "              (lora_dropout): ModuleDict(\n",
1595
              "                (default): Dropout(p=0.05, inplace=False)\n",
1596
              "              )\n",
1597
              "              (lora_A): ModuleDict(\n",
1598
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1599
              "              )\n",
1600
              "              (lora_B): ModuleDict(\n",
1601
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1602
              "              )\n",
1603
              "              (lora_embedding_A): ParameterDict()\n",
1604
              "              (lora_embedding_B): ParameterDict()\n",
1605
              "            )\n",
1606
              "            (inner_attn): SelfAttention(\n",
1607
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1608
              "            )\n",
1609
              "            (inner_cross_attn): CrossAttention(\n",
1610
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1611
              "            )\n",
1612
              "          )\n",
1613
              "          (mlp): MLP(\n",
1614
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1615
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1616
              "            (act): NewGELUActivation()\n",
1617
              "          )\n",
1618
              "        )\n",
1619
              "        (9): ParallelBlock(\n",
1620
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1621
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1622
              "          (mixer): MHA(\n",
1623
              "            (rotary_emb): RotaryEmbedding()\n",
1624
              "            (Wqkv): Linear4bit(\n",
1625
              "              in_features=2048, out_features=6144, bias=True\n",
1626
              "              (lora_dropout): ModuleDict(\n",
1627
              "                (default): Dropout(p=0.05, inplace=False)\n",
1628
              "              )\n",
1629
              "              (lora_A): ModuleDict(\n",
1630
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1631
              "              )\n",
1632
              "              (lora_B): ModuleDict(\n",
1633
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1634
              "              )\n",
1635
              "              (lora_embedding_A): ParameterDict()\n",
1636
              "              (lora_embedding_B): ParameterDict()\n",
1637
              "            )\n",
1638
              "            (out_proj): Linear4bit(\n",
1639
              "              in_features=2048, out_features=2048, bias=True\n",
1640
              "              (lora_dropout): ModuleDict(\n",
1641
              "                (default): Dropout(p=0.05, inplace=False)\n",
1642
              "              )\n",
1643
              "              (lora_A): ModuleDict(\n",
1644
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1645
              "              )\n",
1646
              "              (lora_B): ModuleDict(\n",
1647
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1648
              "              )\n",
1649
              "              (lora_embedding_A): ParameterDict()\n",
1650
              "              (lora_embedding_B): ParameterDict()\n",
1651
              "            )\n",
1652
              "            (inner_attn): SelfAttention(\n",
1653
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1654
              "            )\n",
1655
              "            (inner_cross_attn): CrossAttention(\n",
1656
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1657
              "            )\n",
1658
              "          )\n",
1659
              "          (mlp): MLP(\n",
1660
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1661
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1662
              "            (act): NewGELUActivation()\n",
1663
              "          )\n",
1664
              "        )\n",
1665
              "        (10): ParallelBlock(\n",
1666
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1667
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1668
              "          (mixer): MHA(\n",
1669
              "            (rotary_emb): RotaryEmbedding()\n",
1670
              "            (Wqkv): Linear4bit(\n",
1671
              "              in_features=2048, out_features=6144, bias=True\n",
1672
              "              (lora_dropout): ModuleDict(\n",
1673
              "                (default): Dropout(p=0.05, inplace=False)\n",
1674
              "              )\n",
1675
              "              (lora_A): ModuleDict(\n",
1676
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1677
              "              )\n",
1678
              "              (lora_B): ModuleDict(\n",
1679
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1680
              "              )\n",
1681
              "              (lora_embedding_A): ParameterDict()\n",
1682
              "              (lora_embedding_B): ParameterDict()\n",
1683
              "            )\n",
1684
              "            (out_proj): Linear4bit(\n",
1685
              "              in_features=2048, out_features=2048, bias=True\n",
1686
              "              (lora_dropout): ModuleDict(\n",
1687
              "                (default): Dropout(p=0.05, inplace=False)\n",
1688
              "              )\n",
1689
              "              (lora_A): ModuleDict(\n",
1690
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1691
              "              )\n",
1692
              "              (lora_B): ModuleDict(\n",
1693
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1694
              "              )\n",
1695
              "              (lora_embedding_A): ParameterDict()\n",
1696
              "              (lora_embedding_B): ParameterDict()\n",
1697
              "            )\n",
1698
              "            (inner_attn): SelfAttention(\n",
1699
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1700
              "            )\n",
1701
              "            (inner_cross_attn): CrossAttention(\n",
1702
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1703
              "            )\n",
1704
              "          )\n",
1705
              "          (mlp): MLP(\n",
1706
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1707
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1708
              "            (act): NewGELUActivation()\n",
1709
              "          )\n",
1710
              "        )\n",
1711
              "        (11): ParallelBlock(\n",
1712
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1713
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1714
              "          (mixer): MHA(\n",
1715
              "            (rotary_emb): RotaryEmbedding()\n",
1716
              "            (Wqkv): Linear4bit(\n",
1717
              "              in_features=2048, out_features=6144, bias=True\n",
1718
              "              (lora_dropout): ModuleDict(\n",
1719
              "                (default): Dropout(p=0.05, inplace=False)\n",
1720
              "              )\n",
1721
              "              (lora_A): ModuleDict(\n",
1722
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1723
              "              )\n",
1724
              "              (lora_B): ModuleDict(\n",
1725
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1726
              "              )\n",
1727
              "              (lora_embedding_A): ParameterDict()\n",
1728
              "              (lora_embedding_B): ParameterDict()\n",
1729
              "            )\n",
1730
              "            (out_proj): Linear4bit(\n",
1731
              "              in_features=2048, out_features=2048, bias=True\n",
1732
              "              (lora_dropout): ModuleDict(\n",
1733
              "                (default): Dropout(p=0.05, inplace=False)\n",
1734
              "              )\n",
1735
              "              (lora_A): ModuleDict(\n",
1736
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1737
              "              )\n",
1738
              "              (lora_B): ModuleDict(\n",
1739
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1740
              "              )\n",
1741
              "              (lora_embedding_A): ParameterDict()\n",
1742
              "              (lora_embedding_B): ParameterDict()\n",
1743
              "            )\n",
1744
              "            (inner_attn): SelfAttention(\n",
1745
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1746
              "            )\n",
1747
              "            (inner_cross_attn): CrossAttention(\n",
1748
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1749
              "            )\n",
1750
              "          )\n",
1751
              "          (mlp): MLP(\n",
1752
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1753
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1754
              "            (act): NewGELUActivation()\n",
1755
              "          )\n",
1756
              "        )\n",
1757
              "        (12): ParallelBlock(\n",
1758
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1759
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1760
              "          (mixer): MHA(\n",
1761
              "            (rotary_emb): RotaryEmbedding()\n",
1762
              "            (Wqkv): Linear4bit(\n",
1763
              "              in_features=2048, out_features=6144, bias=True\n",
1764
              "              (lora_dropout): ModuleDict(\n",
1765
              "                (default): Dropout(p=0.05, inplace=False)\n",
1766
              "              )\n",
1767
              "              (lora_A): ModuleDict(\n",
1768
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1769
              "              )\n",
1770
              "              (lora_B): ModuleDict(\n",
1771
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1772
              "              )\n",
1773
              "              (lora_embedding_A): ParameterDict()\n",
1774
              "              (lora_embedding_B): ParameterDict()\n",
1775
              "            )\n",
1776
              "            (out_proj): Linear4bit(\n",
1777
              "              in_features=2048, out_features=2048, bias=True\n",
1778
              "              (lora_dropout): ModuleDict(\n",
1779
              "                (default): Dropout(p=0.05, inplace=False)\n",
1780
              "              )\n",
1781
              "              (lora_A): ModuleDict(\n",
1782
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1783
              "              )\n",
1784
              "              (lora_B): ModuleDict(\n",
1785
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1786
              "              )\n",
1787
              "              (lora_embedding_A): ParameterDict()\n",
1788
              "              (lora_embedding_B): ParameterDict()\n",
1789
              "            )\n",
1790
              "            (inner_attn): SelfAttention(\n",
1791
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1792
              "            )\n",
1793
              "            (inner_cross_attn): CrossAttention(\n",
1794
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1795
              "            )\n",
1796
              "          )\n",
1797
              "          (mlp): MLP(\n",
1798
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1799
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1800
              "            (act): NewGELUActivation()\n",
1801
              "          )\n",
1802
              "        )\n",
1803
              "        (13): ParallelBlock(\n",
1804
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1805
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1806
              "          (mixer): MHA(\n",
1807
              "            (rotary_emb): RotaryEmbedding()\n",
1808
              "            (Wqkv): Linear4bit(\n",
1809
              "              in_features=2048, out_features=6144, bias=True\n",
1810
              "              (lora_dropout): ModuleDict(\n",
1811
              "                (default): Dropout(p=0.05, inplace=False)\n",
1812
              "              )\n",
1813
              "              (lora_A): ModuleDict(\n",
1814
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1815
              "              )\n",
1816
              "              (lora_B): ModuleDict(\n",
1817
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1818
              "              )\n",
1819
              "              (lora_embedding_A): ParameterDict()\n",
1820
              "              (lora_embedding_B): ParameterDict()\n",
1821
              "            )\n",
1822
              "            (out_proj): Linear4bit(\n",
1823
              "              in_features=2048, out_features=2048, bias=True\n",
1824
              "              (lora_dropout): ModuleDict(\n",
1825
              "                (default): Dropout(p=0.05, inplace=False)\n",
1826
              "              )\n",
1827
              "              (lora_A): ModuleDict(\n",
1828
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1829
              "              )\n",
1830
              "              (lora_B): ModuleDict(\n",
1831
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1832
              "              )\n",
1833
              "              (lora_embedding_A): ParameterDict()\n",
1834
              "              (lora_embedding_B): ParameterDict()\n",
1835
              "            )\n",
1836
              "            (inner_attn): SelfAttention(\n",
1837
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1838
              "            )\n",
1839
              "            (inner_cross_attn): CrossAttention(\n",
1840
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1841
              "            )\n",
1842
              "          )\n",
1843
              "          (mlp): MLP(\n",
1844
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1845
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1846
              "            (act): NewGELUActivation()\n",
1847
              "          )\n",
1848
              "        )\n",
1849
              "        (14): ParallelBlock(\n",
1850
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1851
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1852
              "          (mixer): MHA(\n",
1853
              "            (rotary_emb): RotaryEmbedding()\n",
1854
              "            (Wqkv): Linear4bit(\n",
1855
              "              in_features=2048, out_features=6144, bias=True\n",
1856
              "              (lora_dropout): ModuleDict(\n",
1857
              "                (default): Dropout(p=0.05, inplace=False)\n",
1858
              "              )\n",
1859
              "              (lora_A): ModuleDict(\n",
1860
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1861
              "              )\n",
1862
              "              (lora_B): ModuleDict(\n",
1863
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1864
              "              )\n",
1865
              "              (lora_embedding_A): ParameterDict()\n",
1866
              "              (lora_embedding_B): ParameterDict()\n",
1867
              "            )\n",
1868
              "            (out_proj): Linear4bit(\n",
1869
              "              in_features=2048, out_features=2048, bias=True\n",
1870
              "              (lora_dropout): ModuleDict(\n",
1871
              "                (default): Dropout(p=0.05, inplace=False)\n",
1872
              "              )\n",
1873
              "              (lora_A): ModuleDict(\n",
1874
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1875
              "              )\n",
1876
              "              (lora_B): ModuleDict(\n",
1877
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1878
              "              )\n",
1879
              "              (lora_embedding_A): ParameterDict()\n",
1880
              "              (lora_embedding_B): ParameterDict()\n",
1881
              "            )\n",
1882
              "            (inner_attn): SelfAttention(\n",
1883
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1884
              "            )\n",
1885
              "            (inner_cross_attn): CrossAttention(\n",
1886
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1887
              "            )\n",
1888
              "          )\n",
1889
              "          (mlp): MLP(\n",
1890
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1891
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1892
              "            (act): NewGELUActivation()\n",
1893
              "          )\n",
1894
              "        )\n",
1895
              "        (15): ParallelBlock(\n",
1896
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1897
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1898
              "          (mixer): MHA(\n",
1899
              "            (rotary_emb): RotaryEmbedding()\n",
1900
              "            (Wqkv): Linear4bit(\n",
1901
              "              in_features=2048, out_features=6144, bias=True\n",
1902
              "              (lora_dropout): ModuleDict(\n",
1903
              "                (default): Dropout(p=0.05, inplace=False)\n",
1904
              "              )\n",
1905
              "              (lora_A): ModuleDict(\n",
1906
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1907
              "              )\n",
1908
              "              (lora_B): ModuleDict(\n",
1909
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1910
              "              )\n",
1911
              "              (lora_embedding_A): ParameterDict()\n",
1912
              "              (lora_embedding_B): ParameterDict()\n",
1913
              "            )\n",
1914
              "            (out_proj): Linear4bit(\n",
1915
              "              in_features=2048, out_features=2048, bias=True\n",
1916
              "              (lora_dropout): ModuleDict(\n",
1917
              "                (default): Dropout(p=0.05, inplace=False)\n",
1918
              "              )\n",
1919
              "              (lora_A): ModuleDict(\n",
1920
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1921
              "              )\n",
1922
              "              (lora_B): ModuleDict(\n",
1923
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1924
              "              )\n",
1925
              "              (lora_embedding_A): ParameterDict()\n",
1926
              "              (lora_embedding_B): ParameterDict()\n",
1927
              "            )\n",
1928
              "            (inner_attn): SelfAttention(\n",
1929
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1930
              "            )\n",
1931
              "            (inner_cross_attn): CrossAttention(\n",
1932
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1933
              "            )\n",
1934
              "          )\n",
1935
              "          (mlp): MLP(\n",
1936
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1937
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1938
              "            (act): NewGELUActivation()\n",
1939
              "          )\n",
1940
              "        )\n",
1941
              "        (16): ParallelBlock(\n",
1942
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1943
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1944
              "          (mixer): MHA(\n",
1945
              "            (rotary_emb): RotaryEmbedding()\n",
1946
              "            (Wqkv): Linear4bit(\n",
1947
              "              in_features=2048, out_features=6144, bias=True\n",
1948
              "              (lora_dropout): ModuleDict(\n",
1949
              "                (default): Dropout(p=0.05, inplace=False)\n",
1950
              "              )\n",
1951
              "              (lora_A): ModuleDict(\n",
1952
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1953
              "              )\n",
1954
              "              (lora_B): ModuleDict(\n",
1955
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
1956
              "              )\n",
1957
              "              (lora_embedding_A): ParameterDict()\n",
1958
              "              (lora_embedding_B): ParameterDict()\n",
1959
              "            )\n",
1960
              "            (out_proj): Linear4bit(\n",
1961
              "              in_features=2048, out_features=2048, bias=True\n",
1962
              "              (lora_dropout): ModuleDict(\n",
1963
              "                (default): Dropout(p=0.05, inplace=False)\n",
1964
              "              )\n",
1965
              "              (lora_A): ModuleDict(\n",
1966
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1967
              "              )\n",
1968
              "              (lora_B): ModuleDict(\n",
1969
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
1970
              "              )\n",
1971
              "              (lora_embedding_A): ParameterDict()\n",
1972
              "              (lora_embedding_B): ParameterDict()\n",
1973
              "            )\n",
1974
              "            (inner_attn): SelfAttention(\n",
1975
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1976
              "            )\n",
1977
              "            (inner_cross_attn): CrossAttention(\n",
1978
              "              (drop): Dropout(p=0.0, inplace=False)\n",
1979
              "            )\n",
1980
              "          )\n",
1981
              "          (mlp): MLP(\n",
1982
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
1983
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
1984
              "            (act): NewGELUActivation()\n",
1985
              "          )\n",
1986
              "        )\n",
1987
              "        (17): ParallelBlock(\n",
1988
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
1989
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
1990
              "          (mixer): MHA(\n",
1991
              "            (rotary_emb): RotaryEmbedding()\n",
1992
              "            (Wqkv): Linear4bit(\n",
1993
              "              in_features=2048, out_features=6144, bias=True\n",
1994
              "              (lora_dropout): ModuleDict(\n",
1995
              "                (default): Dropout(p=0.05, inplace=False)\n",
1996
              "              )\n",
1997
              "              (lora_A): ModuleDict(\n",
1998
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
1999
              "              )\n",
2000
              "              (lora_B): ModuleDict(\n",
2001
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2002
              "              )\n",
2003
              "              (lora_embedding_A): ParameterDict()\n",
2004
              "              (lora_embedding_B): ParameterDict()\n",
2005
              "            )\n",
2006
              "            (out_proj): Linear4bit(\n",
2007
              "              in_features=2048, out_features=2048, bias=True\n",
2008
              "              (lora_dropout): ModuleDict(\n",
2009
              "                (default): Dropout(p=0.05, inplace=False)\n",
2010
              "              )\n",
2011
              "              (lora_A): ModuleDict(\n",
2012
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2013
              "              )\n",
2014
              "              (lora_B): ModuleDict(\n",
2015
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2016
              "              )\n",
2017
              "              (lora_embedding_A): ParameterDict()\n",
2018
              "              (lora_embedding_B): ParameterDict()\n",
2019
              "            )\n",
2020
              "            (inner_attn): SelfAttention(\n",
2021
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2022
              "            )\n",
2023
              "            (inner_cross_attn): CrossAttention(\n",
2024
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2025
              "            )\n",
2026
              "          )\n",
2027
              "          (mlp): MLP(\n",
2028
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2029
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2030
              "            (act): NewGELUActivation()\n",
2031
              "          )\n",
2032
              "        )\n",
2033
              "        (18): ParallelBlock(\n",
2034
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2035
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2036
              "          (mixer): MHA(\n",
2037
              "            (rotary_emb): RotaryEmbedding()\n",
2038
              "            (Wqkv): Linear4bit(\n",
2039
              "              in_features=2048, out_features=6144, bias=True\n",
2040
              "              (lora_dropout): ModuleDict(\n",
2041
              "                (default): Dropout(p=0.05, inplace=False)\n",
2042
              "              )\n",
2043
              "              (lora_A): ModuleDict(\n",
2044
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2045
              "              )\n",
2046
              "              (lora_B): ModuleDict(\n",
2047
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2048
              "              )\n",
2049
              "              (lora_embedding_A): ParameterDict()\n",
2050
              "              (lora_embedding_B): ParameterDict()\n",
2051
              "            )\n",
2052
              "            (out_proj): Linear4bit(\n",
2053
              "              in_features=2048, out_features=2048, bias=True\n",
2054
              "              (lora_dropout): ModuleDict(\n",
2055
              "                (default): Dropout(p=0.05, inplace=False)\n",
2056
              "              )\n",
2057
              "              (lora_A): ModuleDict(\n",
2058
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2059
              "              )\n",
2060
              "              (lora_B): ModuleDict(\n",
2061
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2062
              "              )\n",
2063
              "              (lora_embedding_A): ParameterDict()\n",
2064
              "              (lora_embedding_B): ParameterDict()\n",
2065
              "            )\n",
2066
              "            (inner_attn): SelfAttention(\n",
2067
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2068
              "            )\n",
2069
              "            (inner_cross_attn): CrossAttention(\n",
2070
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2071
              "            )\n",
2072
              "          )\n",
2073
              "          (mlp): MLP(\n",
2074
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2075
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2076
              "            (act): NewGELUActivation()\n",
2077
              "          )\n",
2078
              "        )\n",
2079
              "        (19): ParallelBlock(\n",
2080
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2081
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2082
              "          (mixer): MHA(\n",
2083
              "            (rotary_emb): RotaryEmbedding()\n",
2084
              "            (Wqkv): Linear4bit(\n",
2085
              "              in_features=2048, out_features=6144, bias=True\n",
2086
              "              (lora_dropout): ModuleDict(\n",
2087
              "                (default): Dropout(p=0.05, inplace=False)\n",
2088
              "              )\n",
2089
              "              (lora_A): ModuleDict(\n",
2090
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2091
              "              )\n",
2092
              "              (lora_B): ModuleDict(\n",
2093
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2094
              "              )\n",
2095
              "              (lora_embedding_A): ParameterDict()\n",
2096
              "              (lora_embedding_B): ParameterDict()\n",
2097
              "            )\n",
2098
              "            (out_proj): Linear4bit(\n",
2099
              "              in_features=2048, out_features=2048, bias=True\n",
2100
              "              (lora_dropout): ModuleDict(\n",
2101
              "                (default): Dropout(p=0.05, inplace=False)\n",
2102
              "              )\n",
2103
              "              (lora_A): ModuleDict(\n",
2104
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2105
              "              )\n",
2106
              "              (lora_B): ModuleDict(\n",
2107
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2108
              "              )\n",
2109
              "              (lora_embedding_A): ParameterDict()\n",
2110
              "              (lora_embedding_B): ParameterDict()\n",
2111
              "            )\n",
2112
              "            (inner_attn): SelfAttention(\n",
2113
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2114
              "            )\n",
2115
              "            (inner_cross_attn): CrossAttention(\n",
2116
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2117
              "            )\n",
2118
              "          )\n",
2119
              "          (mlp): MLP(\n",
2120
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2121
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2122
              "            (act): NewGELUActivation()\n",
2123
              "          )\n",
2124
              "        )\n",
2125
              "        (20): ParallelBlock(\n",
2126
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2127
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2128
              "          (mixer): MHA(\n",
2129
              "            (rotary_emb): RotaryEmbedding()\n",
2130
              "            (Wqkv): Linear4bit(\n",
2131
              "              in_features=2048, out_features=6144, bias=True\n",
2132
              "              (lora_dropout): ModuleDict(\n",
2133
              "                (default): Dropout(p=0.05, inplace=False)\n",
2134
              "              )\n",
2135
              "              (lora_A): ModuleDict(\n",
2136
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2137
              "              )\n",
2138
              "              (lora_B): ModuleDict(\n",
2139
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2140
              "              )\n",
2141
              "              (lora_embedding_A): ParameterDict()\n",
2142
              "              (lora_embedding_B): ParameterDict()\n",
2143
              "            )\n",
2144
              "            (out_proj): Linear4bit(\n",
2145
              "              in_features=2048, out_features=2048, bias=True\n",
2146
              "              (lora_dropout): ModuleDict(\n",
2147
              "                (default): Dropout(p=0.05, inplace=False)\n",
2148
              "              )\n",
2149
              "              (lora_A): ModuleDict(\n",
2150
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2151
              "              )\n",
2152
              "              (lora_B): ModuleDict(\n",
2153
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2154
              "              )\n",
2155
              "              (lora_embedding_A): ParameterDict()\n",
2156
              "              (lora_embedding_B): ParameterDict()\n",
2157
              "            )\n",
2158
              "            (inner_attn): SelfAttention(\n",
2159
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2160
              "            )\n",
2161
              "            (inner_cross_attn): CrossAttention(\n",
2162
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2163
              "            )\n",
2164
              "          )\n",
2165
              "          (mlp): MLP(\n",
2166
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2167
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2168
              "            (act): NewGELUActivation()\n",
2169
              "          )\n",
2170
              "        )\n",
2171
              "        (21): ParallelBlock(\n",
2172
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2173
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2174
              "          (mixer): MHA(\n",
2175
              "            (rotary_emb): RotaryEmbedding()\n",
2176
              "            (Wqkv): Linear4bit(\n",
2177
              "              in_features=2048, out_features=6144, bias=True\n",
2178
              "              (lora_dropout): ModuleDict(\n",
2179
              "                (default): Dropout(p=0.05, inplace=False)\n",
2180
              "              )\n",
2181
              "              (lora_A): ModuleDict(\n",
2182
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2183
              "              )\n",
2184
              "              (lora_B): ModuleDict(\n",
2185
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2186
              "              )\n",
2187
              "              (lora_embedding_A): ParameterDict()\n",
2188
              "              (lora_embedding_B): ParameterDict()\n",
2189
              "            )\n",
2190
              "            (out_proj): Linear4bit(\n",
2191
              "              in_features=2048, out_features=2048, bias=True\n",
2192
              "              (lora_dropout): ModuleDict(\n",
2193
              "                (default): Dropout(p=0.05, inplace=False)\n",
2194
              "              )\n",
2195
              "              (lora_A): ModuleDict(\n",
2196
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2197
              "              )\n",
2198
              "              (lora_B): ModuleDict(\n",
2199
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2200
              "              )\n",
2201
              "              (lora_embedding_A): ParameterDict()\n",
2202
              "              (lora_embedding_B): ParameterDict()\n",
2203
              "            )\n",
2204
              "            (inner_attn): SelfAttention(\n",
2205
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2206
              "            )\n",
2207
              "            (inner_cross_attn): CrossAttention(\n",
2208
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2209
              "            )\n",
2210
              "          )\n",
2211
              "          (mlp): MLP(\n",
2212
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2213
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2214
              "            (act): NewGELUActivation()\n",
2215
              "          )\n",
2216
              "        )\n",
2217
              "        (22): ParallelBlock(\n",
2218
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2219
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2220
              "          (mixer): MHA(\n",
2221
              "            (rotary_emb): RotaryEmbedding()\n",
2222
              "            (Wqkv): Linear4bit(\n",
2223
              "              in_features=2048, out_features=6144, bias=True\n",
2224
              "              (lora_dropout): ModuleDict(\n",
2225
              "                (default): Dropout(p=0.05, inplace=False)\n",
2226
              "              )\n",
2227
              "              (lora_A): ModuleDict(\n",
2228
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2229
              "              )\n",
2230
              "              (lora_B): ModuleDict(\n",
2231
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2232
              "              )\n",
2233
              "              (lora_embedding_A): ParameterDict()\n",
2234
              "              (lora_embedding_B): ParameterDict()\n",
2235
              "            )\n",
2236
              "            (out_proj): Linear4bit(\n",
2237
              "              in_features=2048, out_features=2048, bias=True\n",
2238
              "              (lora_dropout): ModuleDict(\n",
2239
              "                (default): Dropout(p=0.05, inplace=False)\n",
2240
              "              )\n",
2241
              "              (lora_A): ModuleDict(\n",
2242
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2243
              "              )\n",
2244
              "              (lora_B): ModuleDict(\n",
2245
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2246
              "              )\n",
2247
              "              (lora_embedding_A): ParameterDict()\n",
2248
              "              (lora_embedding_B): ParameterDict()\n",
2249
              "            )\n",
2250
              "            (inner_attn): SelfAttention(\n",
2251
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2252
              "            )\n",
2253
              "            (inner_cross_attn): CrossAttention(\n",
2254
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2255
              "            )\n",
2256
              "          )\n",
2257
              "          (mlp): MLP(\n",
2258
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2259
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2260
              "            (act): NewGELUActivation()\n",
2261
              "          )\n",
2262
              "        )\n",
2263
              "        (23): ParallelBlock(\n",
2264
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2265
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2266
              "          (mixer): MHA(\n",
2267
              "            (rotary_emb): RotaryEmbedding()\n",
2268
              "            (Wqkv): Linear4bit(\n",
2269
              "              in_features=2048, out_features=6144, bias=True\n",
2270
              "              (lora_dropout): ModuleDict(\n",
2271
              "                (default): Dropout(p=0.05, inplace=False)\n",
2272
              "              )\n",
2273
              "              (lora_A): ModuleDict(\n",
2274
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2275
              "              )\n",
2276
              "              (lora_B): ModuleDict(\n",
2277
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2278
              "              )\n",
2279
              "              (lora_embedding_A): ParameterDict()\n",
2280
              "              (lora_embedding_B): ParameterDict()\n",
2281
              "            )\n",
2282
              "            (out_proj): Linear4bit(\n",
2283
              "              in_features=2048, out_features=2048, bias=True\n",
2284
              "              (lora_dropout): ModuleDict(\n",
2285
              "                (default): Dropout(p=0.05, inplace=False)\n",
2286
              "              )\n",
2287
              "              (lora_A): ModuleDict(\n",
2288
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2289
              "              )\n",
2290
              "              (lora_B): ModuleDict(\n",
2291
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2292
              "              )\n",
2293
              "              (lora_embedding_A): ParameterDict()\n",
2294
              "              (lora_embedding_B): ParameterDict()\n",
2295
              "            )\n",
2296
              "            (inner_attn): SelfAttention(\n",
2297
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2298
              "            )\n",
2299
              "            (inner_cross_attn): CrossAttention(\n",
2300
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2301
              "            )\n",
2302
              "          )\n",
2303
              "          (mlp): MLP(\n",
2304
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2305
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2306
              "            (act): NewGELUActivation()\n",
2307
              "          )\n",
2308
              "        )\n",
2309
              "        (24): ParallelBlock(\n",
2310
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2311
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2312
              "          (mixer): MHA(\n",
2313
              "            (rotary_emb): RotaryEmbedding()\n",
2314
              "            (Wqkv): Linear4bit(\n",
2315
              "              in_features=2048, out_features=6144, bias=True\n",
2316
              "              (lora_dropout): ModuleDict(\n",
2317
              "                (default): Dropout(p=0.05, inplace=False)\n",
2318
              "              )\n",
2319
              "              (lora_A): ModuleDict(\n",
2320
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2321
              "              )\n",
2322
              "              (lora_B): ModuleDict(\n",
2323
              "                (default): Linear(in_features=16, out_features=6144, bias=False)\n",
2324
              "              )\n",
2325
              "              (lora_embedding_A): ParameterDict()\n",
2326
              "              (lora_embedding_B): ParameterDict()\n",
2327
              "            )\n",
2328
              "            (out_proj): Linear4bit(\n",
2329
              "              in_features=2048, out_features=2048, bias=True\n",
2330
              "              (lora_dropout): ModuleDict(\n",
2331
              "                (default): Dropout(p=0.05, inplace=False)\n",
2332
              "              )\n",
2333
              "              (lora_A): ModuleDict(\n",
2334
              "                (default): Linear(in_features=2048, out_features=16, bias=False)\n",
2335
              "              )\n",
2336
              "              (lora_B): ModuleDict(\n",
2337
              "                (default): Linear(in_features=16, out_features=2048, bias=False)\n",
2338
              "              )\n",
2339
              "              (lora_embedding_A): ParameterDict()\n",
2340
              "              (lora_embedding_B): ParameterDict()\n",
2341
              "            )\n",
2342
              "            (inner_attn): SelfAttention(\n",
2343
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2344
              "            )\n",
2345
              "            (inner_cross_attn): CrossAttention(\n",
2346
              "              (drop): Dropout(p=0.0, inplace=False)\n",
2347
              "            )\n",
2348
              "          )\n",
2349
              "          (mlp): MLP(\n",
2350
              "            (fc1): Linear4bit(in_features=2048, out_features=8192, bias=True)\n",
2351
              "            (fc2): Linear4bit(in_features=8192, out_features=2048, bias=True)\n",
2352
              "            (act): NewGELUActivation()\n",
2353
              "          )\n",
2354
              "        )\n",
2355
              "        (25): CausalLMHead(\n",
2356
              "          (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2357
              "          (linear): Linear(in_features=2048, out_features=51200, bias=True)\n",
2358
              "        )\n",
2359
              "      )\n",
2360
              "      (loss): CausalLMLoss(\n",
2361
              "        (loss_fct): CrossEntropyLoss()\n",
2362
              "      )\n",
2363
              "    )\n",
2364
              "  )\n",
2365
              ")"
2366
            ]
2367
          },
2368
          "metadata": {},
2369
          "execution_count": 30
2370
        }
2371
      ],
2372
      "source": [
2373
        "trainer.model"
2374
      ]
2375
    },
2376
    {
2377
      "cell_type": "markdown",
2378
      "source": [
2379
        "## Inference"
2380
      ],
2381
      "metadata": {
2382
        "id": "7lSxtsWehGcy"
2383
      }
2384
    },
2385
    {
2386
      "cell_type": "code",
2387
      "source": [
2388
        "from peft import PeftModel\n",
2389
        "from transformers import AutoModelForCausalLM\n",
2390
        "import torch\n",
2391
        "model = AutoModelForCausalLM.from_pretrained(\"microsoft/phi-1_5\", trust_remote_code=True, torch_dtype=torch.float32)\n",
2392
        "peft_model = PeftModel.from_pretrained(model, \"ashishpatel26/phi-1_5-finetuned-dialogstudio\", from_transformers=True)\n",
2393
        "model = peft_model.merge_and_unload()\n",
2394
        "model"
2395
      ],
2396
      "metadata": {
2397
        "colab": {
2398
          "base_uri": "https://localhost:8080/",
2399
          "height": 1000,
2400
          "referenced_widgets": [
2401
            "18d7c57661f54a2f906b86e5b92a356d",
2402
            "e132669f57024ebcadc1f0d3899fbda2",
2403
            "f3e78fd1331b4463a243475e12bd2480",
2404
            "386c7bc116b849aaa2bd4c8b0c27b0e9",
2405
            "4385c5f8fc5d4d179e02ebf65aae4e0f",
2406
            "59032aea122142da8744876e7d561bca",
2407
            "b7bf29e09c554482823942f17fb09217",
2408
            "faf28ec7a6304a1d8e52952aff97c6b4",
2409
            "f5ed4018421f446da4ff34741a166d38",
2410
            "1919c4500c88418dad25ed0801033031",
2411
            "c4a29910b33d4a348ceef13239cf835e",
2412
            "31405710033848009cf8227a7611864f",
2413
            "cc862dc1f8884513b8eefb569ab50424",
2414
            "3cd7c95a507d459f86b9eadd7cfdcb85",
2415
            "683da23b78b54a8eb3c1984cd29bbeb2",
2416
            "f022a8aac72d490fb89495f01c929885",
2417
            "ac31ba38940c445e9305623046dc2473",
2418
            "1880da50605c435da03377a6e535d3a6",
2419
            "d645078ee2094f37a1bfbaa6727474bb",
2420
            "a0c4211bdd9e4857b16fbad1d7b1d3e1",
2421
            "3bee676ed0274dee9160f3260f253acf",
2422
            "93909168ffc24b5ba6c03a2a067b99c1"
2423
          ]
2424
        },
2425
        "id": "e43OUEfYOzFF",
2426
        "outputId": "76eec92a-79aa-41af-de41-63b60483470c"
2427
      },
2428
      "execution_count": 1,
2429
      "outputs": [
2430
        {
2431
          "output_type": "display_data",
2432
          "data": {
2433
            "text/plain": [
2434
              "Downloading (…)/adapter_config.json:   0%|          | 0.00/440 [00:00<?, ?B/s]"
2435
            ],
2436
            "application/vnd.jupyter.widget-view+json": {
2437
              "version_major": 2,
2438
              "version_minor": 0,
2439
              "model_id": "18d7c57661f54a2f906b86e5b92a356d"
2440
            }
2441
          },
2442
          "metadata": {}
2443
        },
2444
        {
2445
          "output_type": "display_data",
2446
          "data": {
2447
            "text/plain": [
2448
              "Downloading adapter_model.bin:   0%|          | 0.00/18.9M [00:00<?, ?B/s]"
2449
            ],
2450
            "application/vnd.jupyter.widget-view+json": {
2451
              "version_major": 2,
2452
              "version_minor": 0,
2453
              "model_id": "31405710033848009cf8227a7611864f"
2454
            }
2455
          },
2456
          "metadata": {}
2457
        },
2458
        {
2459
          "output_type": "execute_result",
2460
          "data": {
2461
            "text/plain": [
2462
              "MixFormerSequentialForCausalLM(\n",
2463
              "  (layers): Sequential(\n",
2464
              "    (0): Embedding(\n",
2465
              "      (wte): Embedding(51200, 2048)\n",
2466
              "      (drop): Dropout(p=0.0, inplace=False)\n",
2467
              "    )\n",
2468
              "    (1): ParallelBlock(\n",
2469
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2470
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2471
              "      (mixer): MHA(\n",
2472
              "        (rotary_emb): RotaryEmbedding()\n",
2473
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2474
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2475
              "        (inner_attn): SelfAttention(\n",
2476
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2477
              "        )\n",
2478
              "        (inner_cross_attn): CrossAttention(\n",
2479
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2480
              "        )\n",
2481
              "      )\n",
2482
              "      (mlp): MLP(\n",
2483
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2484
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2485
              "        (act): NewGELUActivation()\n",
2486
              "      )\n",
2487
              "    )\n",
2488
              "    (2): ParallelBlock(\n",
2489
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2490
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2491
              "      (mixer): MHA(\n",
2492
              "        (rotary_emb): RotaryEmbedding()\n",
2493
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2494
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2495
              "        (inner_attn): SelfAttention(\n",
2496
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2497
              "        )\n",
2498
              "        (inner_cross_attn): CrossAttention(\n",
2499
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2500
              "        )\n",
2501
              "      )\n",
2502
              "      (mlp): MLP(\n",
2503
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2504
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2505
              "        (act): NewGELUActivation()\n",
2506
              "      )\n",
2507
              "    )\n",
2508
              "    (3): ParallelBlock(\n",
2509
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2510
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2511
              "      (mixer): MHA(\n",
2512
              "        (rotary_emb): RotaryEmbedding()\n",
2513
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2514
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2515
              "        (inner_attn): SelfAttention(\n",
2516
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2517
              "        )\n",
2518
              "        (inner_cross_attn): CrossAttention(\n",
2519
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2520
              "        )\n",
2521
              "      )\n",
2522
              "      (mlp): MLP(\n",
2523
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2524
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2525
              "        (act): NewGELUActivation()\n",
2526
              "      )\n",
2527
              "    )\n",
2528
              "    (4): ParallelBlock(\n",
2529
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2530
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2531
              "      (mixer): MHA(\n",
2532
              "        (rotary_emb): RotaryEmbedding()\n",
2533
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2534
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2535
              "        (inner_attn): SelfAttention(\n",
2536
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2537
              "        )\n",
2538
              "        (inner_cross_attn): CrossAttention(\n",
2539
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2540
              "        )\n",
2541
              "      )\n",
2542
              "      (mlp): MLP(\n",
2543
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2544
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2545
              "        (act): NewGELUActivation()\n",
2546
              "      )\n",
2547
              "    )\n",
2548
              "    (5): ParallelBlock(\n",
2549
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2550
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2551
              "      (mixer): MHA(\n",
2552
              "        (rotary_emb): RotaryEmbedding()\n",
2553
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2554
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2555
              "        (inner_attn): SelfAttention(\n",
2556
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2557
              "        )\n",
2558
              "        (inner_cross_attn): CrossAttention(\n",
2559
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2560
              "        )\n",
2561
              "      )\n",
2562
              "      (mlp): MLP(\n",
2563
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2564
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2565
              "        (act): NewGELUActivation()\n",
2566
              "      )\n",
2567
              "    )\n",
2568
              "    (6): ParallelBlock(\n",
2569
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2570
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2571
              "      (mixer): MHA(\n",
2572
              "        (rotary_emb): RotaryEmbedding()\n",
2573
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2574
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2575
              "        (inner_attn): SelfAttention(\n",
2576
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2577
              "        )\n",
2578
              "        (inner_cross_attn): CrossAttention(\n",
2579
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2580
              "        )\n",
2581
              "      )\n",
2582
              "      (mlp): MLP(\n",
2583
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2584
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2585
              "        (act): NewGELUActivation()\n",
2586
              "      )\n",
2587
              "    )\n",
2588
              "    (7): ParallelBlock(\n",
2589
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2590
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2591
              "      (mixer): MHA(\n",
2592
              "        (rotary_emb): RotaryEmbedding()\n",
2593
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2594
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2595
              "        (inner_attn): SelfAttention(\n",
2596
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2597
              "        )\n",
2598
              "        (inner_cross_attn): CrossAttention(\n",
2599
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2600
              "        )\n",
2601
              "      )\n",
2602
              "      (mlp): MLP(\n",
2603
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2604
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2605
              "        (act): NewGELUActivation()\n",
2606
              "      )\n",
2607
              "    )\n",
2608
              "    (8): ParallelBlock(\n",
2609
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2610
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2611
              "      (mixer): MHA(\n",
2612
              "        (rotary_emb): RotaryEmbedding()\n",
2613
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2614
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2615
              "        (inner_attn): SelfAttention(\n",
2616
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2617
              "        )\n",
2618
              "        (inner_cross_attn): CrossAttention(\n",
2619
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2620
              "        )\n",
2621
              "      )\n",
2622
              "      (mlp): MLP(\n",
2623
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2624
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2625
              "        (act): NewGELUActivation()\n",
2626
              "      )\n",
2627
              "    )\n",
2628
              "    (9): ParallelBlock(\n",
2629
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2630
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2631
              "      (mixer): MHA(\n",
2632
              "        (rotary_emb): RotaryEmbedding()\n",
2633
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2634
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2635
              "        (inner_attn): SelfAttention(\n",
2636
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2637
              "        )\n",
2638
              "        (inner_cross_attn): CrossAttention(\n",
2639
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2640
              "        )\n",
2641
              "      )\n",
2642
              "      (mlp): MLP(\n",
2643
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2644
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2645
              "        (act): NewGELUActivation()\n",
2646
              "      )\n",
2647
              "    )\n",
2648
              "    (10): ParallelBlock(\n",
2649
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2650
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2651
              "      (mixer): MHA(\n",
2652
              "        (rotary_emb): RotaryEmbedding()\n",
2653
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2654
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2655
              "        (inner_attn): SelfAttention(\n",
2656
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2657
              "        )\n",
2658
              "        (inner_cross_attn): CrossAttention(\n",
2659
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2660
              "        )\n",
2661
              "      )\n",
2662
              "      (mlp): MLP(\n",
2663
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2664
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2665
              "        (act): NewGELUActivation()\n",
2666
              "      )\n",
2667
              "    )\n",
2668
              "    (11): ParallelBlock(\n",
2669
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2670
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2671
              "      (mixer): MHA(\n",
2672
              "        (rotary_emb): RotaryEmbedding()\n",
2673
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2674
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2675
              "        (inner_attn): SelfAttention(\n",
2676
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2677
              "        )\n",
2678
              "        (inner_cross_attn): CrossAttention(\n",
2679
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2680
              "        )\n",
2681
              "      )\n",
2682
              "      (mlp): MLP(\n",
2683
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2684
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2685
              "        (act): NewGELUActivation()\n",
2686
              "      )\n",
2687
              "    )\n",
2688
              "    (12): ParallelBlock(\n",
2689
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2690
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2691
              "      (mixer): MHA(\n",
2692
              "        (rotary_emb): RotaryEmbedding()\n",
2693
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2694
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2695
              "        (inner_attn): SelfAttention(\n",
2696
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2697
              "        )\n",
2698
              "        (inner_cross_attn): CrossAttention(\n",
2699
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2700
              "        )\n",
2701
              "      )\n",
2702
              "      (mlp): MLP(\n",
2703
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2704
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2705
              "        (act): NewGELUActivation()\n",
2706
              "      )\n",
2707
              "    )\n",
2708
              "    (13): ParallelBlock(\n",
2709
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2710
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2711
              "      (mixer): MHA(\n",
2712
              "        (rotary_emb): RotaryEmbedding()\n",
2713
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2714
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2715
              "        (inner_attn): SelfAttention(\n",
2716
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2717
              "        )\n",
2718
              "        (inner_cross_attn): CrossAttention(\n",
2719
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2720
              "        )\n",
2721
              "      )\n",
2722
              "      (mlp): MLP(\n",
2723
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2724
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2725
              "        (act): NewGELUActivation()\n",
2726
              "      )\n",
2727
              "    )\n",
2728
              "    (14): ParallelBlock(\n",
2729
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2730
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2731
              "      (mixer): MHA(\n",
2732
              "        (rotary_emb): RotaryEmbedding()\n",
2733
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2734
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2735
              "        (inner_attn): SelfAttention(\n",
2736
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2737
              "        )\n",
2738
              "        (inner_cross_attn): CrossAttention(\n",
2739
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2740
              "        )\n",
2741
              "      )\n",
2742
              "      (mlp): MLP(\n",
2743
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2744
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2745
              "        (act): NewGELUActivation()\n",
2746
              "      )\n",
2747
              "    )\n",
2748
              "    (15): ParallelBlock(\n",
2749
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2750
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2751
              "      (mixer): MHA(\n",
2752
              "        (rotary_emb): RotaryEmbedding()\n",
2753
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2754
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2755
              "        (inner_attn): SelfAttention(\n",
2756
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2757
              "        )\n",
2758
              "        (inner_cross_attn): CrossAttention(\n",
2759
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2760
              "        )\n",
2761
              "      )\n",
2762
              "      (mlp): MLP(\n",
2763
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2764
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2765
              "        (act): NewGELUActivation()\n",
2766
              "      )\n",
2767
              "    )\n",
2768
              "    (16): ParallelBlock(\n",
2769
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2770
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2771
              "      (mixer): MHA(\n",
2772
              "        (rotary_emb): RotaryEmbedding()\n",
2773
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2774
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2775
              "        (inner_attn): SelfAttention(\n",
2776
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2777
              "        )\n",
2778
              "        (inner_cross_attn): CrossAttention(\n",
2779
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2780
              "        )\n",
2781
              "      )\n",
2782
              "      (mlp): MLP(\n",
2783
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2784
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2785
              "        (act): NewGELUActivation()\n",
2786
              "      )\n",
2787
              "    )\n",
2788
              "    (17): ParallelBlock(\n",
2789
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2790
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2791
              "      (mixer): MHA(\n",
2792
              "        (rotary_emb): RotaryEmbedding()\n",
2793
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2794
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2795
              "        (inner_attn): SelfAttention(\n",
2796
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2797
              "        )\n",
2798
              "        (inner_cross_attn): CrossAttention(\n",
2799
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2800
              "        )\n",
2801
              "      )\n",
2802
              "      (mlp): MLP(\n",
2803
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2804
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2805
              "        (act): NewGELUActivation()\n",
2806
              "      )\n",
2807
              "    )\n",
2808
              "    (18): ParallelBlock(\n",
2809
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2810
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2811
              "      (mixer): MHA(\n",
2812
              "        (rotary_emb): RotaryEmbedding()\n",
2813
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2814
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2815
              "        (inner_attn): SelfAttention(\n",
2816
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2817
              "        )\n",
2818
              "        (inner_cross_attn): CrossAttention(\n",
2819
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2820
              "        )\n",
2821
              "      )\n",
2822
              "      (mlp): MLP(\n",
2823
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2824
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2825
              "        (act): NewGELUActivation()\n",
2826
              "      )\n",
2827
              "    )\n",
2828
              "    (19): ParallelBlock(\n",
2829
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2830
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2831
              "      (mixer): MHA(\n",
2832
              "        (rotary_emb): RotaryEmbedding()\n",
2833
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2834
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2835
              "        (inner_attn): SelfAttention(\n",
2836
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2837
              "        )\n",
2838
              "        (inner_cross_attn): CrossAttention(\n",
2839
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2840
              "        )\n",
2841
              "      )\n",
2842
              "      (mlp): MLP(\n",
2843
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2844
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2845
              "        (act): NewGELUActivation()\n",
2846
              "      )\n",
2847
              "    )\n",
2848
              "    (20): ParallelBlock(\n",
2849
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2850
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2851
              "      (mixer): MHA(\n",
2852
              "        (rotary_emb): RotaryEmbedding()\n",
2853
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2854
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2855
              "        (inner_attn): SelfAttention(\n",
2856
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2857
              "        )\n",
2858
              "        (inner_cross_attn): CrossAttention(\n",
2859
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2860
              "        )\n",
2861
              "      )\n",
2862
              "      (mlp): MLP(\n",
2863
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2864
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2865
              "        (act): NewGELUActivation()\n",
2866
              "      )\n",
2867
              "    )\n",
2868
              "    (21): ParallelBlock(\n",
2869
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2870
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2871
              "      (mixer): MHA(\n",
2872
              "        (rotary_emb): RotaryEmbedding()\n",
2873
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2874
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2875
              "        (inner_attn): SelfAttention(\n",
2876
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2877
              "        )\n",
2878
              "        (inner_cross_attn): CrossAttention(\n",
2879
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2880
              "        )\n",
2881
              "      )\n",
2882
              "      (mlp): MLP(\n",
2883
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2884
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2885
              "        (act): NewGELUActivation()\n",
2886
              "      )\n",
2887
              "    )\n",
2888
              "    (22): ParallelBlock(\n",
2889
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2890
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2891
              "      (mixer): MHA(\n",
2892
              "        (rotary_emb): RotaryEmbedding()\n",
2893
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2894
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2895
              "        (inner_attn): SelfAttention(\n",
2896
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2897
              "        )\n",
2898
              "        (inner_cross_attn): CrossAttention(\n",
2899
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2900
              "        )\n",
2901
              "      )\n",
2902
              "      (mlp): MLP(\n",
2903
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2904
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2905
              "        (act): NewGELUActivation()\n",
2906
              "      )\n",
2907
              "    )\n",
2908
              "    (23): ParallelBlock(\n",
2909
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2910
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2911
              "      (mixer): MHA(\n",
2912
              "        (rotary_emb): RotaryEmbedding()\n",
2913
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2914
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2915
              "        (inner_attn): SelfAttention(\n",
2916
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2917
              "        )\n",
2918
              "        (inner_cross_attn): CrossAttention(\n",
2919
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2920
              "        )\n",
2921
              "      )\n",
2922
              "      (mlp): MLP(\n",
2923
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2924
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2925
              "        (act): NewGELUActivation()\n",
2926
              "      )\n",
2927
              "    )\n",
2928
              "    (24): ParallelBlock(\n",
2929
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2930
              "      (resid_dropout): Dropout(p=0.0, inplace=False)\n",
2931
              "      (mixer): MHA(\n",
2932
              "        (rotary_emb): RotaryEmbedding()\n",
2933
              "        (Wqkv): Linear(in_features=2048, out_features=6144, bias=True)\n",
2934
              "        (out_proj): Linear(in_features=2048, out_features=2048, bias=True)\n",
2935
              "        (inner_attn): SelfAttention(\n",
2936
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2937
              "        )\n",
2938
              "        (inner_cross_attn): CrossAttention(\n",
2939
              "          (drop): Dropout(p=0.0, inplace=False)\n",
2940
              "        )\n",
2941
              "      )\n",
2942
              "      (mlp): MLP(\n",
2943
              "        (fc1): Linear(in_features=2048, out_features=8192, bias=True)\n",
2944
              "        (fc2): Linear(in_features=8192, out_features=2048, bias=True)\n",
2945
              "        (act): NewGELUActivation()\n",
2946
              "      )\n",
2947
              "    )\n",
2948
              "    (25): CausalLMHead(\n",
2949
              "      (ln): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
2950
              "      (linear): Linear(in_features=2048, out_features=51200, bias=True)\n",
2951
              "    )\n",
2952
              "  )\n",
2953
              "  (loss): CausalLMLoss(\n",
2954
              "    (loss_fct): CrossEntropyLoss()\n",
2955
              "  )\n",
2956
              ")"
2957
            ]
2958
          },
2959
          "metadata": {},
2960
          "execution_count": 1
2961
        }
2962
      ]
2963
    },
2964
    {
2965
      "cell_type": "code",
2966
      "source": [
2967
        "import torch\n",
2968
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
2969
        "\n",
2970
        "model = AutoModelForCausalLM.from_pretrained(\"ashishpatel26/phi-1_5-finetuned-dialogstudio\", trust_remote_code=True, torch_dtype=torch.float32)\n",
2971
        "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/phi-1_5\", trust_remote_code=True)\n",
2972
        "inputs = tokenizer(f'''{dataset[\"test\"]['text'][0]}''', return_tensors=\"pt\", return_attention_mask=False)\n",
2973
        "\n",
2974
        "outputs = model.generate(**inputs, max_length=512)\n",
2975
        "text = tokenizer.batch_decode(outputs)[0]\n",
2976
        "print(text)"
2977
      ],
2978
      "metadata": {
2979
        "id": "vpbzxK0zhHk8"
2980
      },
2981
      "execution_count": null,
2982
      "outputs": []
2983
    },
2984
    {
2985
      "cell_type": "code",
2986
      "source": [
2987
        "# dataset[\"test\"]['text'][0]"
2988
      ],
2989
      "metadata": {
2990
        "colab": {
2991
          "base_uri": "https://localhost:8080/",
2992
          "height": 139
2993
        },
2994
        "id": "oU9augJbhm44",
2995
        "outputId": "f85452c7-96a7-47e0-d3e8-52a7b12573aa"
2996
      },
2997
      "execution_count": 19,
2998
      "outputs": [
2999
        {
3000
          "output_type": "execute_result",
3001
          "data": {
3002
            "text/plain": [
3003
              "'### Instruction: Below is a conversation between a human and an AI agent. Write a summary of the conversation.\\n\\n### Input:\\nuser: looking to change my flight Friday, Oct 27. GRMSKV to DL4728 from SLC to ORD. Is that an option and what is the cost? Jess\\nagent: The difference in fare is $185.30. This would include all airport taxes and fees. The ticket is non-refundable changeable with a fee, *ALS and may result in additional fare collection for changes when making a future changes. *ALS\\nuser: I had a first class seat purchased for the original flight, would that be the same with this flight to Chicago?\\nagent: Hello, Jess. That is the fare difference. You will have to call us at 1 800 221 1212 to make any changes. It is in First class. *TAY\\nuser: thx\\nagent: Our pleasure. *ALS\\nuser: Do I have to call or is there a means to do this online?\\nagent: You can call or you can login to your trip on our website to make changes. *TJE\\n\\n### Response:\\nCustomer is looking to change the flight on Friday Oct 27 is that an option and asking about cost. Agent replying that there is an difference in fare and this would include all airport taxes and fees and ticket is non refundable changeable with a fee.'"
3004
            ],
3005
            "application/vnd.google.colaboratory.intrinsic+json": {
3006
              "type": "string"
3007
            }
3008
          },
3009
          "metadata": {},
3010
          "execution_count": 19
3011
        }
3012
      ]
3013
    },
3014
    {
3015
      "cell_type": "markdown",
3016
      "metadata": {
3017
        "id": "pfxoP-IvnheQ"
3018
      },
3019
      "source": [
3020
        "## References\n",
3021
        "\n",
3022
        "- https://huggingface.co/datasets/Salesforce/dialogstudio\n",
3023
        "- https://huggingface.co/meta-llama/Llama-2-7b-hf"
3024
      ]
3025
    }
3026
  ],
3027
  "metadata": {
3028
    "accelerator": "GPU",
3029
    "colab": {
3030
      "provenance": [],
3031
      "gpuType": "T4",
3032
      "include_colab_link": true
3033
    },
3034
    "kernelspec": {
3035
      "display_name": "Python 3",
3036
      "name": "python3"
3037
    },
3038
    "language_info": {
3039
      "name": "python"
3040
    },
3041
    "widgets": {
3042
      "application/vnd.jupyter.widget-state+json": {
3043
        "96ef5be98b5c4f2c9945b1a8c0989d7b": {
3044
          "model_module": "@jupyter-widgets/controls",
3045
          "model_name": "VBoxModel",
3046
          "model_module_version": "1.5.0",
3047
          "state": {
3048
            "_dom_classes": [],
3049
            "_model_module": "@jupyter-widgets/controls",
3050
            "_model_module_version": "1.5.0",
3051
            "_model_name": "VBoxModel",
3052
            "_view_count": null,
3053
            "_view_module": "@jupyter-widgets/controls",
3054
            "_view_module_version": "1.5.0",
3055
            "_view_name": "VBoxView",
3056
            "box_style": "",
3057
            "children": [
3058
              "IPY_MODEL_2cf7de58a08f4a10a0388fd5284adc61",
3059
              "IPY_MODEL_3e55c582114c43d19f9a88819739d9a5",
3060
              "IPY_MODEL_cdd259b528bf4cd7b1c2f834cec0372d",
3061
              "IPY_MODEL_1440bf6528a844f399aea3997b0348ba"
3062
            ],
3063
            "layout": "IPY_MODEL_a68cf26cf964423b8ce413bc6c8c5299"
3064
          }
3065
        },
3066
        "29aca8e4365b4bee99d5292d8e74cdd6": {
3067
          "model_module": "@jupyter-widgets/controls",
3068
          "model_name": "HTMLModel",
3069
          "model_module_version": "1.5.0",
3070
          "state": {
3071
            "_dom_classes": [],
3072
            "_model_module": "@jupyter-widgets/controls",
3073
            "_model_module_version": "1.5.0",
3074
            "_model_name": "HTMLModel",
3075
            "_view_count": null,
3076
            "_view_module": "@jupyter-widgets/controls",
3077
            "_view_module_version": "1.5.0",
3078
            "_view_name": "HTMLView",
3079
            "description": "",
3080
            "description_tooltip": null,
3081
            "layout": "IPY_MODEL_ed2f736177914d2abb1c2efb6ecce6bb",
3082
            "placeholder": "​",
3083
            "style": "IPY_MODEL_6c2ff22a40b543b6ac26ccd2b7298a61",
3084
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
3085
          }
3086
        },
3087
        "c83294f6ba864858a05d5222d84eb0c3": {
3088
          "model_module": "@jupyter-widgets/controls",
3089
          "model_name": "PasswordModel",
3090
          "model_module_version": "1.5.0",
3091
          "state": {
3092
            "_dom_classes": [],
3093
            "_model_module": "@jupyter-widgets/controls",
3094
            "_model_module_version": "1.5.0",
3095
            "_model_name": "PasswordModel",
3096
            "_view_count": null,
3097
            "_view_module": "@jupyter-widgets/controls",
3098
            "_view_module_version": "1.5.0",
3099
            "_view_name": "PasswordView",
3100
            "continuous_update": true,
3101
            "description": "Token:",
3102
            "description_tooltip": null,
3103
            "disabled": false,
3104
            "layout": "IPY_MODEL_bb5fc1bf3f4c465e8b0846f22e71a23b",
3105
            "placeholder": "​",
3106
            "style": "IPY_MODEL_42cae2119e5e49b9b7250a6f96f373bc",
3107
            "value": ""
3108
          }
3109
        },
3110
        "7254587396ba4701835ee666ee3b697b": {
3111
          "model_module": "@jupyter-widgets/controls",
3112
          "model_name": "CheckboxModel",
3113
          "model_module_version": "1.5.0",
3114
          "state": {
3115
            "_dom_classes": [],
3116
            "_model_module": "@jupyter-widgets/controls",
3117
            "_model_module_version": "1.5.0",
3118
            "_model_name": "CheckboxModel",
3119
            "_view_count": null,
3120
            "_view_module": "@jupyter-widgets/controls",
3121
            "_view_module_version": "1.5.0",
3122
            "_view_name": "CheckboxView",
3123
            "description": "Add token as git credential?",
3124
            "description_tooltip": null,
3125
            "disabled": false,
3126
            "indent": true,
3127
            "layout": "IPY_MODEL_8701e3ece5b04773a157dacc8f793374",
3128
            "style": "IPY_MODEL_b3b75ce135e049bbacd598eac3d579ce",
3129
            "value": true
3130
          }
3131
        },
3132
        "896def48ce8148f18a13b0a46d0919b3": {
3133
          "model_module": "@jupyter-widgets/controls",
3134
          "model_name": "ButtonModel",
3135
          "model_module_version": "1.5.0",
3136
          "state": {
3137
            "_dom_classes": [],
3138
            "_model_module": "@jupyter-widgets/controls",
3139
            "_model_module_version": "1.5.0",
3140
            "_model_name": "ButtonModel",
3141
            "_view_count": null,
3142
            "_view_module": "@jupyter-widgets/controls",
3143
            "_view_module_version": "1.5.0",
3144
            "_view_name": "ButtonView",
3145
            "button_style": "",
3146
            "description": "Login",
3147
            "disabled": false,
3148
            "icon": "",
3149
            "layout": "IPY_MODEL_6bb60bf24297418384b709e106c6c71a",
3150
            "style": "IPY_MODEL_4b209751747c4ff98d905702b4ba3ab6",
3151
            "tooltip": ""
3152
          }
3153
        },
3154
        "5bce118ef58b43b6ab295e6272d84668": {
3155
          "model_module": "@jupyter-widgets/controls",
3156
          "model_name": "HTMLModel",
3157
          "model_module_version": "1.5.0",
3158
          "state": {
3159
            "_dom_classes": [],
3160
            "_model_module": "@jupyter-widgets/controls",
3161
            "_model_module_version": "1.5.0",
3162
            "_model_name": "HTMLModel",
3163
            "_view_count": null,
3164
            "_view_module": "@jupyter-widgets/controls",
3165
            "_view_module_version": "1.5.0",
3166
            "_view_name": "HTMLView",
3167
            "description": "",
3168
            "description_tooltip": null,
3169
            "layout": "IPY_MODEL_ca3fcc5229864c2d8030b8e1bce3680a",
3170
            "placeholder": "​",
3171
            "style": "IPY_MODEL_e43a98b2e5ba41a5a483c9320f982960",
3172
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
3173
          }
3174
        },
3175
        "a68cf26cf964423b8ce413bc6c8c5299": {
3176
          "model_module": "@jupyter-widgets/base",
3177
          "model_name": "LayoutModel",
3178
          "model_module_version": "1.2.0",
3179
          "state": {
3180
            "_model_module": "@jupyter-widgets/base",
3181
            "_model_module_version": "1.2.0",
3182
            "_model_name": "LayoutModel",
3183
            "_view_count": null,
3184
            "_view_module": "@jupyter-widgets/base",
3185
            "_view_module_version": "1.2.0",
3186
            "_view_name": "LayoutView",
3187
            "align_content": null,
3188
            "align_items": "center",
3189
            "align_self": null,
3190
            "border": null,
3191
            "bottom": null,
3192
            "display": "flex",
3193
            "flex": null,
3194
            "flex_flow": "column",
3195
            "grid_area": null,
3196
            "grid_auto_columns": null,
3197
            "grid_auto_flow": null,
3198
            "grid_auto_rows": null,
3199
            "grid_column": null,
3200
            "grid_gap": null,
3201
            "grid_row": null,
3202
            "grid_template_areas": null,
3203
            "grid_template_columns": null,
3204
            "grid_template_rows": null,
3205
            "height": null,
3206
            "justify_content": null,
3207
            "justify_items": null,
3208
            "left": null,
3209
            "margin": null,
3210
            "max_height": null,
3211
            "max_width": null,
3212
            "min_height": null,
3213
            "min_width": null,
3214
            "object_fit": null,
3215
            "object_position": null,
3216
            "order": null,
3217
            "overflow": null,
3218
            "overflow_x": null,
3219
            "overflow_y": null,
3220
            "padding": null,
3221
            "right": null,
3222
            "top": null,
3223
            "visibility": null,
3224
            "width": "50%"
3225
          }
3226
        },
3227
        "ed2f736177914d2abb1c2efb6ecce6bb": {
3228
          "model_module": "@jupyter-widgets/base",
3229
          "model_name": "LayoutModel",
3230
          "model_module_version": "1.2.0",
3231
          "state": {
3232
            "_model_module": "@jupyter-widgets/base",
3233
            "_model_module_version": "1.2.0",
3234
            "_model_name": "LayoutModel",
3235
            "_view_count": null,
3236
            "_view_module": "@jupyter-widgets/base",
3237
            "_view_module_version": "1.2.0",
3238
            "_view_name": "LayoutView",
3239
            "align_content": null,
3240
            "align_items": null,
3241
            "align_self": null,
3242
            "border": null,
3243
            "bottom": null,
3244
            "display": null,
3245
            "flex": null,
3246
            "flex_flow": null,
3247
            "grid_area": null,
3248
            "grid_auto_columns": null,
3249
            "grid_auto_flow": null,
3250
            "grid_auto_rows": null,
3251
            "grid_column": null,
3252
            "grid_gap": null,
3253
            "grid_row": null,
3254
            "grid_template_areas": null,
3255
            "grid_template_columns": null,
3256
            "grid_template_rows": null,
3257
            "height": null,
3258
            "justify_content": null,
3259
            "justify_items": null,
3260
            "left": null,
3261
            "margin": null,
3262
            "max_height": null,
3263
            "max_width": null,
3264
            "min_height": null,
3265
            "min_width": null,
3266
            "object_fit": null,
3267
            "object_position": null,
3268
            "order": null,
3269
            "overflow": null,
3270
            "overflow_x": null,
3271
            "overflow_y": null,
3272
            "padding": null,
3273
            "right": null,
3274
            "top": null,
3275
            "visibility": null,
3276
            "width": null
3277
          }
3278
        },
3279
        "6c2ff22a40b543b6ac26ccd2b7298a61": {
3280
          "model_module": "@jupyter-widgets/controls",
3281
          "model_name": "DescriptionStyleModel",
3282
          "model_module_version": "1.5.0",
3283
          "state": {
3284
            "_model_module": "@jupyter-widgets/controls",
3285
            "_model_module_version": "1.5.0",
3286
            "_model_name": "DescriptionStyleModel",
3287
            "_view_count": null,
3288
            "_view_module": "@jupyter-widgets/base",
3289
            "_view_module_version": "1.2.0",
3290
            "_view_name": "StyleView",
3291
            "description_width": ""
3292
          }
3293
        },
3294
        "bb5fc1bf3f4c465e8b0846f22e71a23b": {
3295
          "model_module": "@jupyter-widgets/base",
3296
          "model_name": "LayoutModel",
3297
          "model_module_version": "1.2.0",
3298
          "state": {
3299
            "_model_module": "@jupyter-widgets/base",
3300
            "_model_module_version": "1.2.0",
3301
            "_model_name": "LayoutModel",
3302
            "_view_count": null,
3303
            "_view_module": "@jupyter-widgets/base",
3304
            "_view_module_version": "1.2.0",
3305
            "_view_name": "LayoutView",
3306
            "align_content": null,
3307
            "align_items": null,
3308
            "align_self": null,
3309
            "border": null,
3310
            "bottom": null,
3311
            "display": null,
3312
            "flex": null,
3313
            "flex_flow": null,
3314
            "grid_area": null,
3315
            "grid_auto_columns": null,
3316
            "grid_auto_flow": null,
3317
            "grid_auto_rows": null,
3318
            "grid_column": null,
3319
            "grid_gap": null,
3320
            "grid_row": null,
3321
            "grid_template_areas": null,
3322
            "grid_template_columns": null,
3323
            "grid_template_rows": null,
3324
            "height": null,
3325
            "justify_content": null,
3326
            "justify_items": null,
3327
            "left": null,
3328
            "margin": null,
3329
            "max_height": null,
3330
            "max_width": null,
3331
            "min_height": null,
3332
            "min_width": null,
3333
            "object_fit": null,
3334
            "object_position": null,
3335
            "order": null,
3336
            "overflow": null,
3337
            "overflow_x": null,
3338
            "overflow_y": null,
3339
            "padding": null,
3340
            "right": null,
3341
            "top": null,
3342
            "visibility": null,
3343
            "width": null
3344
          }
3345
        },
3346
        "42cae2119e5e49b9b7250a6f96f373bc": {
3347
          "model_module": "@jupyter-widgets/controls",
3348
          "model_name": "DescriptionStyleModel",
3349
          "model_module_version": "1.5.0",
3350
          "state": {
3351
            "_model_module": "@jupyter-widgets/controls",
3352
            "_model_module_version": "1.5.0",
3353
            "_model_name": "DescriptionStyleModel",
3354
            "_view_count": null,
3355
            "_view_module": "@jupyter-widgets/base",
3356
            "_view_module_version": "1.2.0",
3357
            "_view_name": "StyleView",
3358
            "description_width": ""
3359
          }
3360
        },
3361
        "8701e3ece5b04773a157dacc8f793374": {
3362
          "model_module": "@jupyter-widgets/base",
3363
          "model_name": "LayoutModel",
3364
          "model_module_version": "1.2.0",
3365
          "state": {
3366
            "_model_module": "@jupyter-widgets/base",
3367
            "_model_module_version": "1.2.0",
3368
            "_model_name": "LayoutModel",
3369
            "_view_count": null,
3370
            "_view_module": "@jupyter-widgets/base",
3371
            "_view_module_version": "1.2.0",
3372
            "_view_name": "LayoutView",
3373
            "align_content": null,
3374
            "align_items": null,
3375
            "align_self": null,
3376
            "border": null,
3377
            "bottom": null,
3378
            "display": null,
3379
            "flex": null,
3380
            "flex_flow": null,
3381
            "grid_area": null,
3382
            "grid_auto_columns": null,
3383
            "grid_auto_flow": null,
3384
            "grid_auto_rows": null,
3385
            "grid_column": null,
3386
            "grid_gap": null,
3387
            "grid_row": null,
3388
            "grid_template_areas": null,
3389
            "grid_template_columns": null,
3390
            "grid_template_rows": null,
3391
            "height": null,
3392
            "justify_content": null,
3393
            "justify_items": null,
3394
            "left": null,
3395
            "margin": null,
3396
            "max_height": null,
3397
            "max_width": null,
3398
            "min_height": null,
3399
            "min_width": null,
3400
            "object_fit": null,
3401
            "object_position": null,
3402
            "order": null,
3403
            "overflow": null,
3404
            "overflow_x": null,
3405
            "overflow_y": null,
3406
            "padding": null,
3407
            "right": null,
3408
            "top": null,
3409
            "visibility": null,
3410
            "width": null
3411
          }
3412
        },
3413
        "b3b75ce135e049bbacd598eac3d579ce": {
3414
          "model_module": "@jupyter-widgets/controls",
3415
          "model_name": "DescriptionStyleModel",
3416
          "model_module_version": "1.5.0",
3417
          "state": {
3418
            "_model_module": "@jupyter-widgets/controls",
3419
            "_model_module_version": "1.5.0",
3420
            "_model_name": "DescriptionStyleModel",
3421
            "_view_count": null,
3422
            "_view_module": "@jupyter-widgets/base",
3423
            "_view_module_version": "1.2.0",
3424
            "_view_name": "StyleView",
3425
            "description_width": ""
3426
          }
3427
        },
3428
        "6bb60bf24297418384b709e106c6c71a": {
3429
          "model_module": "@jupyter-widgets/base",
3430
          "model_name": "LayoutModel",
3431
          "model_module_version": "1.2.0",
3432
          "state": {
3433
            "_model_module": "@jupyter-widgets/base",
3434
            "_model_module_version": "1.2.0",
3435
            "_model_name": "LayoutModel",
3436
            "_view_count": null,
3437
            "_view_module": "@jupyter-widgets/base",
3438
            "_view_module_version": "1.2.0",
3439
            "_view_name": "LayoutView",
3440
            "align_content": null,
3441
            "align_items": null,
3442
            "align_self": null,
3443
            "border": null,
3444
            "bottom": null,
3445
            "display": null,
3446
            "flex": null,
3447
            "flex_flow": null,
3448
            "grid_area": null,
3449
            "grid_auto_columns": null,
3450
            "grid_auto_flow": null,
3451
            "grid_auto_rows": null,
3452
            "grid_column": null,
3453
            "grid_gap": null,
3454
            "grid_row": null,
3455
            "grid_template_areas": null,
3456
            "grid_template_columns": null,
3457
            "grid_template_rows": null,
3458
            "height": null,
3459
            "justify_content": null,
3460
            "justify_items": null,
3461
            "left": null,
3462
            "margin": null,
3463
            "max_height": null,
3464
            "max_width": null,
3465
            "min_height": null,
3466
            "min_width": null,
3467
            "object_fit": null,
3468
            "object_position": null,
3469
            "order": null,
3470
            "overflow": null,
3471
            "overflow_x": null,
3472
            "overflow_y": null,
3473
            "padding": null,
3474
            "right": null,
3475
            "top": null,
3476
            "visibility": null,
3477
            "width": null
3478
          }
3479
        },
3480
        "4b209751747c4ff98d905702b4ba3ab6": {
3481
          "model_module": "@jupyter-widgets/controls",
3482
          "model_name": "ButtonStyleModel",
3483
          "model_module_version": "1.5.0",
3484
          "state": {
3485
            "_model_module": "@jupyter-widgets/controls",
3486
            "_model_module_version": "1.5.0",
3487
            "_model_name": "ButtonStyleModel",
3488
            "_view_count": null,
3489
            "_view_module": "@jupyter-widgets/base",
3490
            "_view_module_version": "1.2.0",
3491
            "_view_name": "StyleView",
3492
            "button_color": null,
3493
            "font_weight": ""
3494
          }
3495
        },
3496
        "ca3fcc5229864c2d8030b8e1bce3680a": {
3497
          "model_module": "@jupyter-widgets/base",
3498
          "model_name": "LayoutModel",
3499
          "model_module_version": "1.2.0",
3500
          "state": {
3501
            "_model_module": "@jupyter-widgets/base",
3502
            "_model_module_version": "1.2.0",
3503
            "_model_name": "LayoutModel",
3504
            "_view_count": null,
3505
            "_view_module": "@jupyter-widgets/base",
3506
            "_view_module_version": "1.2.0",
3507
            "_view_name": "LayoutView",
3508
            "align_content": null,
3509
            "align_items": null,
3510
            "align_self": null,
3511
            "border": null,
3512
            "bottom": null,
3513
            "display": null,
3514
            "flex": null,
3515
            "flex_flow": null,
3516
            "grid_area": null,
3517
            "grid_auto_columns": null,
3518
            "grid_auto_flow": null,
3519
            "grid_auto_rows": null,
3520
            "grid_column": null,
3521
            "grid_gap": null,
3522
            "grid_row": null,
3523
            "grid_template_areas": null,
3524
            "grid_template_columns": null,
3525
            "grid_template_rows": null,
3526
            "height": null,
3527
            "justify_content": null,
3528
            "justify_items": null,
3529
            "left": null,
3530
            "margin": null,
3531
            "max_height": null,
3532
            "max_width": null,
3533
            "min_height": null,
3534
            "min_width": null,
3535
            "object_fit": null,
3536
            "object_position": null,
3537
            "order": null,
3538
            "overflow": null,
3539
            "overflow_x": null,
3540
            "overflow_y": null,
3541
            "padding": null,
3542
            "right": null,
3543
            "top": null,
3544
            "visibility": null,
3545
            "width": null
3546
          }
3547
        },
3548
        "e43a98b2e5ba41a5a483c9320f982960": {
3549
          "model_module": "@jupyter-widgets/controls",
3550
          "model_name": "DescriptionStyleModel",
3551
          "model_module_version": "1.5.0",
3552
          "state": {
3553
            "_model_module": "@jupyter-widgets/controls",
3554
            "_model_module_version": "1.5.0",
3555
            "_model_name": "DescriptionStyleModel",
3556
            "_view_count": null,
3557
            "_view_module": "@jupyter-widgets/base",
3558
            "_view_module_version": "1.2.0",
3559
            "_view_name": "StyleView",
3560
            "description_width": ""
3561
          }
3562
        },
3563
        "f18eecc689234406a32b478a88379011": {
3564
          "model_module": "@jupyter-widgets/controls",
3565
          "model_name": "LabelModel",
3566
          "model_module_version": "1.5.0",
3567
          "state": {
3568
            "_dom_classes": [],
3569
            "_model_module": "@jupyter-widgets/controls",
3570
            "_model_module_version": "1.5.0",
3571
            "_model_name": "LabelModel",
3572
            "_view_count": null,
3573
            "_view_module": "@jupyter-widgets/controls",
3574
            "_view_module_version": "1.5.0",
3575
            "_view_name": "LabelView",
3576
            "description": "",
3577
            "description_tooltip": null,
3578
            "layout": "IPY_MODEL_dc6a98f4481a4d158ffd66d036d9d391",
3579
            "placeholder": "​",
3580
            "style": "IPY_MODEL_c3e8d37c64af4b1eaed0569fd32c3841",
3581
            "value": "Connecting..."
3582
          }
3583
        },
3584
        "dc6a98f4481a4d158ffd66d036d9d391": {
3585
          "model_module": "@jupyter-widgets/base",
3586
          "model_name": "LayoutModel",
3587
          "model_module_version": "1.2.0",
3588
          "state": {
3589
            "_model_module": "@jupyter-widgets/base",
3590
            "_model_module_version": "1.2.0",
3591
            "_model_name": "LayoutModel",
3592
            "_view_count": null,
3593
            "_view_module": "@jupyter-widgets/base",
3594
            "_view_module_version": "1.2.0",
3595
            "_view_name": "LayoutView",
3596
            "align_content": null,
3597
            "align_items": null,
3598
            "align_self": null,
3599
            "border": null,
3600
            "bottom": null,
3601
            "display": null,
3602
            "flex": null,
3603
            "flex_flow": null,
3604
            "grid_area": null,
3605
            "grid_auto_columns": null,
3606
            "grid_auto_flow": null,
3607
            "grid_auto_rows": null,
3608
            "grid_column": null,
3609
            "grid_gap": null,
3610
            "grid_row": null,
3611
            "grid_template_areas": null,
3612
            "grid_template_columns": null,
3613
            "grid_template_rows": null,
3614
            "height": null,
3615
            "justify_content": null,
3616
            "justify_items": null,
3617
            "left": null,
3618
            "margin": null,
3619
            "max_height": null,
3620
            "max_width": null,
3621
            "min_height": null,
3622
            "min_width": null,
3623
            "object_fit": null,
3624
            "object_position": null,
3625
            "order": null,
3626
            "overflow": null,
3627
            "overflow_x": null,
3628
            "overflow_y": null,
3629
            "padding": null,
3630
            "right": null,
3631
            "top": null,
3632
            "visibility": null,
3633
            "width": null
3634
          }
3635
        },
3636
        "c3e8d37c64af4b1eaed0569fd32c3841": {
3637
          "model_module": "@jupyter-widgets/controls",
3638
          "model_name": "DescriptionStyleModel",
3639
          "model_module_version": "1.5.0",
3640
          "state": {
3641
            "_model_module": "@jupyter-widgets/controls",
3642
            "_model_module_version": "1.5.0",
3643
            "_model_name": "DescriptionStyleModel",
3644
            "_view_count": null,
3645
            "_view_module": "@jupyter-widgets/base",
3646
            "_view_module_version": "1.2.0",
3647
            "_view_name": "StyleView",
3648
            "description_width": ""
3649
          }
3650
        },
3651
        "2cf7de58a08f4a10a0388fd5284adc61": {
3652
          "model_module": "@jupyter-widgets/controls",
3653
          "model_name": "LabelModel",
3654
          "model_module_version": "1.5.0",
3655
          "state": {
3656
            "_dom_classes": [],
3657
            "_model_module": "@jupyter-widgets/controls",
3658
            "_model_module_version": "1.5.0",
3659
            "_model_name": "LabelModel",
3660
            "_view_count": null,
3661
            "_view_module": "@jupyter-widgets/controls",
3662
            "_view_module_version": "1.5.0",
3663
            "_view_name": "LabelView",
3664
            "description": "",
3665
            "description_tooltip": null,
3666
            "layout": "IPY_MODEL_60af265316074adab78687ac53070910",
3667
            "placeholder": "​",
3668
            "style": "IPY_MODEL_1c9d8afeb2644c65909f6e689e1af0af",
3669
            "value": "Token is valid (permission: write)."
3670
          }
3671
        },
3672
        "3e55c582114c43d19f9a88819739d9a5": {
3673
          "model_module": "@jupyter-widgets/controls",
3674
          "model_name": "LabelModel",
3675
          "model_module_version": "1.5.0",
3676
          "state": {
3677
            "_dom_classes": [],
3678
            "_model_module": "@jupyter-widgets/controls",
3679
            "_model_module_version": "1.5.0",
3680
            "_model_name": "LabelModel",
3681
            "_view_count": null,
3682
            "_view_module": "@jupyter-widgets/controls",
3683
            "_view_module_version": "1.5.0",
3684
            "_view_name": "LabelView",
3685
            "description": "",
3686
            "description_tooltip": null,
3687
            "layout": "IPY_MODEL_06d7ce85d9784f58add0583feda74530",
3688
            "placeholder": "​",
3689
            "style": "IPY_MODEL_bcacfb2e6c464423b5e8bc0aef675fdd",
3690
            "value": "Your token has been saved in your configured git credential helpers (store)."
3691
          }
3692
        },
3693
        "cdd259b528bf4cd7b1c2f834cec0372d": {
3694
          "model_module": "@jupyter-widgets/controls",
3695
          "model_name": "LabelModel",
3696
          "model_module_version": "1.5.0",
3697
          "state": {
3698
            "_dom_classes": [],
3699
            "_model_module": "@jupyter-widgets/controls",
3700
            "_model_module_version": "1.5.0",
3701
            "_model_name": "LabelModel",
3702
            "_view_count": null,
3703
            "_view_module": "@jupyter-widgets/controls",
3704
            "_view_module_version": "1.5.0",
3705
            "_view_name": "LabelView",
3706
            "description": "",
3707
            "description_tooltip": null,
3708
            "layout": "IPY_MODEL_88c5cc8d66d8408cbe2bfedb365723f3",
3709
            "placeholder": "​",
3710
            "style": "IPY_MODEL_814bac58a4974f70b17b5e19f88b042f",
3711
            "value": "Your token has been saved to /root/.cache/huggingface/token"
3712
          }
3713
        },
3714
        "1440bf6528a844f399aea3997b0348ba": {
3715
          "model_module": "@jupyter-widgets/controls",
3716
          "model_name": "LabelModel",
3717
          "model_module_version": "1.5.0",
3718
          "state": {
3719
            "_dom_classes": [],
3720
            "_model_module": "@jupyter-widgets/controls",
3721
            "_model_module_version": "1.5.0",
3722
            "_model_name": "LabelModel",
3723
            "_view_count": null,
3724
            "_view_module": "@jupyter-widgets/controls",
3725
            "_view_module_version": "1.5.0",
3726
            "_view_name": "LabelView",
3727
            "description": "",
3728
            "description_tooltip": null,
3729
            "layout": "IPY_MODEL_3502bced8abc4cfc87300881857c4e08",
3730
            "placeholder": "​",
3731
            "style": "IPY_MODEL_c726307cdc3c4598ad45879f3f29e60c",
3732
            "value": "Login successful"
3733
          }
3734
        },
3735
        "60af265316074adab78687ac53070910": {
3736
          "model_module": "@jupyter-widgets/base",
3737
          "model_name": "LayoutModel",
3738
          "model_module_version": "1.2.0",
3739
          "state": {
3740
            "_model_module": "@jupyter-widgets/base",
3741
            "_model_module_version": "1.2.0",
3742
            "_model_name": "LayoutModel",
3743
            "_view_count": null,
3744
            "_view_module": "@jupyter-widgets/base",
3745
            "_view_module_version": "1.2.0",
3746
            "_view_name": "LayoutView",
3747
            "align_content": null,
3748
            "align_items": null,
3749
            "align_self": null,
3750
            "border": null,
3751
            "bottom": null,
3752
            "display": null,
3753
            "flex": null,
3754
            "flex_flow": null,
3755
            "grid_area": null,
3756
            "grid_auto_columns": null,
3757
            "grid_auto_flow": null,
3758
            "grid_auto_rows": null,
3759
            "grid_column": null,
3760
            "grid_gap": null,
3761
            "grid_row": null,
3762
            "grid_template_areas": null,
3763
            "grid_template_columns": null,
3764
            "grid_template_rows": null,
3765
            "height": null,
3766
            "justify_content": null,
3767
            "justify_items": null,
3768
            "left": null,
3769
            "margin": null,
3770
            "max_height": null,
3771
            "max_width": null,
3772
            "min_height": null,
3773
            "min_width": null,
3774
            "object_fit": null,
3775
            "object_position": null,
3776
            "order": null,
3777
            "overflow": null,
3778
            "overflow_x": null,
3779
            "overflow_y": null,
3780
            "padding": null,
3781
            "right": null,
3782
            "top": null,
3783
            "visibility": null,
3784
            "width": null
3785
          }
3786
        },
3787
        "1c9d8afeb2644c65909f6e689e1af0af": {
3788
          "model_module": "@jupyter-widgets/controls",
3789
          "model_name": "DescriptionStyleModel",
3790
          "model_module_version": "1.5.0",
3791
          "state": {
3792
            "_model_module": "@jupyter-widgets/controls",
3793
            "_model_module_version": "1.5.0",
3794
            "_model_name": "DescriptionStyleModel",
3795
            "_view_count": null,
3796
            "_view_module": "@jupyter-widgets/base",
3797
            "_view_module_version": "1.2.0",
3798
            "_view_name": "StyleView",
3799
            "description_width": ""
3800
          }
3801
        },
3802
        "06d7ce85d9784f58add0583feda74530": {
3803
          "model_module": "@jupyter-widgets/base",
3804
          "model_name": "LayoutModel",
3805
          "model_module_version": "1.2.0",
3806
          "state": {
3807
            "_model_module": "@jupyter-widgets/base",
3808
            "_model_module_version": "1.2.0",
3809
            "_model_name": "LayoutModel",
3810
            "_view_count": null,
3811
            "_view_module": "@jupyter-widgets/base",
3812
            "_view_module_version": "1.2.0",
3813
            "_view_name": "LayoutView",
3814
            "align_content": null,
3815
            "align_items": null,
3816
            "align_self": null,
3817
            "border": null,
3818
            "bottom": null,
3819
            "display": null,
3820
            "flex": null,
3821
            "flex_flow": null,
3822
            "grid_area": null,
3823
            "grid_auto_columns": null,
3824
            "grid_auto_flow": null,
3825
            "grid_auto_rows": null,
3826
            "grid_column": null,
3827
            "grid_gap": null,
3828
            "grid_row": null,
3829
            "grid_template_areas": null,
3830
            "grid_template_columns": null,
3831
            "grid_template_rows": null,
3832
            "height": null,
3833
            "justify_content": null,
3834
            "justify_items": null,
3835
            "left": null,
3836
            "margin": null,
3837
            "max_height": null,
3838
            "max_width": null,
3839
            "min_height": null,
3840
            "min_width": null,
3841
            "object_fit": null,
3842
            "object_position": null,
3843
            "order": null,
3844
            "overflow": null,
3845
            "overflow_x": null,
3846
            "overflow_y": null,
3847
            "padding": null,
3848
            "right": null,
3849
            "top": null,
3850
            "visibility": null,
3851
            "width": null
3852
          }
3853
        },
3854
        "bcacfb2e6c464423b5e8bc0aef675fdd": {
3855
          "model_module": "@jupyter-widgets/controls",
3856
          "model_name": "DescriptionStyleModel",
3857
          "model_module_version": "1.5.0",
3858
          "state": {
3859
            "_model_module": "@jupyter-widgets/controls",
3860
            "_model_module_version": "1.5.0",
3861
            "_model_name": "DescriptionStyleModel",
3862
            "_view_count": null,
3863
            "_view_module": "@jupyter-widgets/base",
3864
            "_view_module_version": "1.2.0",
3865
            "_view_name": "StyleView",
3866
            "description_width": ""
3867
          }
3868
        },
3869
        "88c5cc8d66d8408cbe2bfedb365723f3": {
3870
          "model_module": "@jupyter-widgets/base",
3871
          "model_name": "LayoutModel",
3872
          "model_module_version": "1.2.0",
3873
          "state": {
3874
            "_model_module": "@jupyter-widgets/base",
3875
            "_model_module_version": "1.2.0",
3876
            "_model_name": "LayoutModel",
3877
            "_view_count": null,
3878
            "_view_module": "@jupyter-widgets/base",
3879
            "_view_module_version": "1.2.0",
3880
            "_view_name": "LayoutView",
3881
            "align_content": null,
3882
            "align_items": null,
3883
            "align_self": null,
3884
            "border": null,
3885
            "bottom": null,
3886
            "display": null,
3887
            "flex": null,
3888
            "flex_flow": null,
3889
            "grid_area": null,
3890
            "grid_auto_columns": null,
3891
            "grid_auto_flow": null,
3892
            "grid_auto_rows": null,
3893
            "grid_column": null,
3894
            "grid_gap": null,
3895
            "grid_row": null,
3896
            "grid_template_areas": null,
3897
            "grid_template_columns": null,
3898
            "grid_template_rows": null,
3899
            "height": null,
3900
            "justify_content": null,
3901
            "justify_items": null,
3902
            "left": null,
3903
            "margin": null,
3904
            "max_height": null,
3905
            "max_width": null,
3906
            "min_height": null,
3907
            "min_width": null,
3908
            "object_fit": null,
3909
            "object_position": null,
3910
            "order": null,
3911
            "overflow": null,
3912
            "overflow_x": null,
3913
            "overflow_y": null,
3914
            "padding": null,
3915
            "right": null,
3916
            "top": null,
3917
            "visibility": null,
3918
            "width": null
3919
          }
3920
        },
3921
        "814bac58a4974f70b17b5e19f88b042f": {
3922
          "model_module": "@jupyter-widgets/controls",
3923
          "model_name": "DescriptionStyleModel",
3924
          "model_module_version": "1.5.0",
3925
          "state": {
3926
            "_model_module": "@jupyter-widgets/controls",
3927
            "_model_module_version": "1.5.0",
3928
            "_model_name": "DescriptionStyleModel",
3929
            "_view_count": null,
3930
            "_view_module": "@jupyter-widgets/base",
3931
            "_view_module_version": "1.2.0",
3932
            "_view_name": "StyleView",
3933
            "description_width": ""
3934
          }
3935
        },
3936
        "3502bced8abc4cfc87300881857c4e08": {
3937
          "model_module": "@jupyter-widgets/base",
3938
          "model_name": "LayoutModel",
3939
          "model_module_version": "1.2.0",
3940
          "state": {
3941
            "_model_module": "@jupyter-widgets/base",
3942
            "_model_module_version": "1.2.0",
3943
            "_model_name": "LayoutModel",
3944
            "_view_count": null,
3945
            "_view_module": "@jupyter-widgets/base",
3946
            "_view_module_version": "1.2.0",
3947
            "_view_name": "LayoutView",
3948
            "align_content": null,
3949
            "align_items": null,
3950
            "align_self": null,
3951
            "border": null,
3952
            "bottom": null,
3953
            "display": null,
3954
            "flex": null,
3955
            "flex_flow": null,
3956
            "grid_area": null,
3957
            "grid_auto_columns": null,
3958
            "grid_auto_flow": null,
3959
            "grid_auto_rows": null,
3960
            "grid_column": null,
3961
            "grid_gap": null,
3962
            "grid_row": null,
3963
            "grid_template_areas": null,
3964
            "grid_template_columns": null,
3965
            "grid_template_rows": null,
3966
            "height": null,
3967
            "justify_content": null,
3968
            "justify_items": null,
3969
            "left": null,
3970
            "margin": null,
3971
            "max_height": null,
3972
            "max_width": null,
3973
            "min_height": null,
3974
            "min_width": null,
3975
            "object_fit": null,
3976
            "object_position": null,
3977
            "order": null,
3978
            "overflow": null,
3979
            "overflow_x": null,
3980
            "overflow_y": null,
3981
            "padding": null,
3982
            "right": null,
3983
            "top": null,
3984
            "visibility": null,
3985
            "width": null
3986
          }
3987
        },
3988
        "c726307cdc3c4598ad45879f3f29e60c": {
3989
          "model_module": "@jupyter-widgets/controls",
3990
          "model_name": "DescriptionStyleModel",
3991
          "model_module_version": "1.5.0",
3992
          "state": {
3993
            "_model_module": "@jupyter-widgets/controls",
3994
            "_model_module_version": "1.5.0",
3995
            "_model_name": "DescriptionStyleModel",
3996
            "_view_count": null,
3997
            "_view_module": "@jupyter-widgets/base",
3998
            "_view_module_version": "1.2.0",
3999
            "_view_name": "StyleView",
4000
            "description_width": ""
4001
          }
4002
        },
4003
        "ee3793650a24458c8f1c9da6f6cf7930": {
4004
          "model_module": "@jupyter-widgets/controls",
4005
          "model_name": "HBoxModel",
4006
          "model_module_version": "1.5.0",
4007
          "state": {
4008
            "_dom_classes": [],
4009
            "_model_module": "@jupyter-widgets/controls",
4010
            "_model_module_version": "1.5.0",
4011
            "_model_name": "HBoxModel",
4012
            "_view_count": null,
4013
            "_view_module": "@jupyter-widgets/controls",
4014
            "_view_module_version": "1.5.0",
4015
            "_view_name": "HBoxView",
4016
            "box_style": "",
4017
            "children": [
4018
              "IPY_MODEL_db88e2bb13fa425aaccc2f7a32f2a946",
4019
              "IPY_MODEL_9f1b94961d43444697b621f5f2a80392",
4020
              "IPY_MODEL_bfa4594314d6437994f2cb57311d97a6"
4021
            ],
4022
            "layout": "IPY_MODEL_dd13edfc70174149b25a15b0858ba61c"
4023
          }
4024
        },
4025
        "db88e2bb13fa425aaccc2f7a32f2a946": {
4026
          "model_module": "@jupyter-widgets/controls",
4027
          "model_name": "HTMLModel",
4028
          "model_module_version": "1.5.0",
4029
          "state": {
4030
            "_dom_classes": [],
4031
            "_model_module": "@jupyter-widgets/controls",
4032
            "_model_module_version": "1.5.0",
4033
            "_model_name": "HTMLModel",
4034
            "_view_count": null,
4035
            "_view_module": "@jupyter-widgets/controls",
4036
            "_view_module_version": "1.5.0",
4037
            "_view_name": "HTMLView",
4038
            "description": "",
4039
            "description_tooltip": null,
4040
            "layout": "IPY_MODEL_2e0685af66bb4e52b72b8e0159df3531",
4041
            "placeholder": "​",
4042
            "style": "IPY_MODEL_0d86661bee2d497fa856892c6b6f3f46",
4043
            "value": "Downloading (…)lve/main/config.json: 100%"
4044
          }
4045
        },
4046
        "9f1b94961d43444697b621f5f2a80392": {
4047
          "model_module": "@jupyter-widgets/controls",
4048
          "model_name": "FloatProgressModel",
4049
          "model_module_version": "1.5.0",
4050
          "state": {
4051
            "_dom_classes": [],
4052
            "_model_module": "@jupyter-widgets/controls",
4053
            "_model_module_version": "1.5.0",
4054
            "_model_name": "FloatProgressModel",
4055
            "_view_count": null,
4056
            "_view_module": "@jupyter-widgets/controls",
4057
            "_view_module_version": "1.5.0",
4058
            "_view_name": "ProgressView",
4059
            "bar_style": "success",
4060
            "description": "",
4061
            "description_tooltip": null,
4062
            "layout": "IPY_MODEL_94916f236b0d4e8a8733871e84df1147",
4063
            "max": 880,
4064
            "min": 0,
4065
            "orientation": "horizontal",
4066
            "style": "IPY_MODEL_97d6e28ba91e45cd899719bb5bc2b6ab",
4067
            "value": 880
4068
          }
4069
        },
4070
        "bfa4594314d6437994f2cb57311d97a6": {
4071
          "model_module": "@jupyter-widgets/controls",
4072
          "model_name": "HTMLModel",
4073
          "model_module_version": "1.5.0",
4074
          "state": {
4075
            "_dom_classes": [],
4076
            "_model_module": "@jupyter-widgets/controls",
4077
            "_model_module_version": "1.5.0",
4078
            "_model_name": "HTMLModel",
4079
            "_view_count": null,
4080
            "_view_module": "@jupyter-widgets/controls",
4081
            "_view_module_version": "1.5.0",
4082
            "_view_name": "HTMLView",
4083
            "description": "",
4084
            "description_tooltip": null,
4085
            "layout": "IPY_MODEL_118f1b7073b847ef987036cd82499d0b",
4086
            "placeholder": "​",
4087
            "style": "IPY_MODEL_dd77325d1d8d417eb6544ae0cbc15133",
4088
            "value": " 880/880 [00:00&lt;00:00, 20.8kB/s]"
4089
          }
4090
        },
4091
        "dd13edfc70174149b25a15b0858ba61c": {
4092
          "model_module": "@jupyter-widgets/base",
4093
          "model_name": "LayoutModel",
4094
          "model_module_version": "1.2.0",
4095
          "state": {
4096
            "_model_module": "@jupyter-widgets/base",
4097
            "_model_module_version": "1.2.0",
4098
            "_model_name": "LayoutModel",
4099
            "_view_count": null,
4100
            "_view_module": "@jupyter-widgets/base",
4101
            "_view_module_version": "1.2.0",
4102
            "_view_name": "LayoutView",
4103
            "align_content": null,
4104
            "align_items": null,
4105
            "align_self": null,
4106
            "border": null,
4107
            "bottom": null,
4108
            "display": null,
4109
            "flex": null,
4110
            "flex_flow": null,
4111
            "grid_area": null,
4112
            "grid_auto_columns": null,
4113
            "grid_auto_flow": null,
4114
            "grid_auto_rows": null,
4115
            "grid_column": null,
4116
            "grid_gap": null,
4117
            "grid_row": null,
4118
            "grid_template_areas": null,
4119
            "grid_template_columns": null,
4120
            "grid_template_rows": null,
4121
            "height": null,
4122
            "justify_content": null,
4123
            "justify_items": null,
4124
            "left": null,
4125
            "margin": null,
4126
            "max_height": null,
4127
            "max_width": null,
4128
            "min_height": null,
4129
            "min_width": null,
4130
            "object_fit": null,
4131
            "object_position": null,
4132
            "order": null,
4133
            "overflow": null,
4134
            "overflow_x": null,
4135
            "overflow_y": null,
4136
            "padding": null,
4137
            "right": null,
4138
            "top": null,
4139
            "visibility": null,
4140
            "width": null
4141
          }
4142
        },
4143
        "2e0685af66bb4e52b72b8e0159df3531": {
4144
          "model_module": "@jupyter-widgets/base",
4145
          "model_name": "LayoutModel",
4146
          "model_module_version": "1.2.0",
4147
          "state": {
4148
            "_model_module": "@jupyter-widgets/base",
4149
            "_model_module_version": "1.2.0",
4150
            "_model_name": "LayoutModel",
4151
            "_view_count": null,
4152
            "_view_module": "@jupyter-widgets/base",
4153
            "_view_module_version": "1.2.0",
4154
            "_view_name": "LayoutView",
4155
            "align_content": null,
4156
            "align_items": null,
4157
            "align_self": null,
4158
            "border": null,
4159
            "bottom": null,
4160
            "display": null,
4161
            "flex": null,
4162
            "flex_flow": null,
4163
            "grid_area": null,
4164
            "grid_auto_columns": null,
4165
            "grid_auto_flow": null,
4166
            "grid_auto_rows": null,
4167
            "grid_column": null,
4168
            "grid_gap": null,
4169
            "grid_row": null,
4170
            "grid_template_areas": null,
4171
            "grid_template_columns": null,
4172
            "grid_template_rows": null,
4173
            "height": null,
4174
            "justify_content": null,
4175
            "justify_items": null,
4176
            "left": null,
4177
            "margin": null,
4178
            "max_height": null,
4179
            "max_width": null,
4180
            "min_height": null,
4181
            "min_width": null,
4182
            "object_fit": null,
4183
            "object_position": null,
4184
            "order": null,
4185
            "overflow": null,
4186
            "overflow_x": null,
4187
            "overflow_y": null,
4188
            "padding": null,
4189
            "right": null,
4190
            "top": null,
4191
            "visibility": null,
4192
            "width": null
4193
          }
4194
        },
4195
        "0d86661bee2d497fa856892c6b6f3f46": {
4196
          "model_module": "@jupyter-widgets/controls",
4197
          "model_name": "DescriptionStyleModel",
4198
          "model_module_version": "1.5.0",
4199
          "state": {
4200
            "_model_module": "@jupyter-widgets/controls",
4201
            "_model_module_version": "1.5.0",
4202
            "_model_name": "DescriptionStyleModel",
4203
            "_view_count": null,
4204
            "_view_module": "@jupyter-widgets/base",
4205
            "_view_module_version": "1.2.0",
4206
            "_view_name": "StyleView",
4207
            "description_width": ""
4208
          }
4209
        },
4210
        "94916f236b0d4e8a8733871e84df1147": {
4211
          "model_module": "@jupyter-widgets/base",
4212
          "model_name": "LayoutModel",
4213
          "model_module_version": "1.2.0",
4214
          "state": {
4215
            "_model_module": "@jupyter-widgets/base",
4216
            "_model_module_version": "1.2.0",
4217
            "_model_name": "LayoutModel",
4218
            "_view_count": null,
4219
            "_view_module": "@jupyter-widgets/base",
4220
            "_view_module_version": "1.2.0",
4221
            "_view_name": "LayoutView",
4222
            "align_content": null,
4223
            "align_items": null,
4224
            "align_self": null,
4225
            "border": null,
4226
            "bottom": null,
4227
            "display": null,
4228
            "flex": null,
4229
            "flex_flow": null,
4230
            "grid_area": null,
4231
            "grid_auto_columns": null,
4232
            "grid_auto_flow": null,
4233
            "grid_auto_rows": null,
4234
            "grid_column": null,
4235
            "grid_gap": null,
4236
            "grid_row": null,
4237
            "grid_template_areas": null,
4238
            "grid_template_columns": null,
4239
            "grid_template_rows": null,
4240
            "height": null,
4241
            "justify_content": null,
4242
            "justify_items": null,
4243
            "left": null,
4244
            "margin": null,
4245
            "max_height": null,
4246
            "max_width": null,
4247
            "min_height": null,
4248
            "min_width": null,
4249
            "object_fit": null,
4250
            "object_position": null,
4251
            "order": null,
4252
            "overflow": null,
4253
            "overflow_x": null,
4254
            "overflow_y": null,
4255
            "padding": null,
4256
            "right": null,
4257
            "top": null,
4258
            "visibility": null,
4259
            "width": null
4260
          }
4261
        },
4262
        "97d6e28ba91e45cd899719bb5bc2b6ab": {
4263
          "model_module": "@jupyter-widgets/controls",
4264
          "model_name": "ProgressStyleModel",
4265
          "model_module_version": "1.5.0",
4266
          "state": {
4267
            "_model_module": "@jupyter-widgets/controls",
4268
            "_model_module_version": "1.5.0",
4269
            "_model_name": "ProgressStyleModel",
4270
            "_view_count": null,
4271
            "_view_module": "@jupyter-widgets/base",
4272
            "_view_module_version": "1.2.0",
4273
            "_view_name": "StyleView",
4274
            "bar_color": null,
4275
            "description_width": ""
4276
          }
4277
        },
4278
        "118f1b7073b847ef987036cd82499d0b": {
4279
          "model_module": "@jupyter-widgets/base",
4280
          "model_name": "LayoutModel",
4281
          "model_module_version": "1.2.0",
4282
          "state": {
4283
            "_model_module": "@jupyter-widgets/base",
4284
            "_model_module_version": "1.2.0",
4285
            "_model_name": "LayoutModel",
4286
            "_view_count": null,
4287
            "_view_module": "@jupyter-widgets/base",
4288
            "_view_module_version": "1.2.0",
4289
            "_view_name": "LayoutView",
4290
            "align_content": null,
4291
            "align_items": null,
4292
            "align_self": null,
4293
            "border": null,
4294
            "bottom": null,
4295
            "display": null,
4296
            "flex": null,
4297
            "flex_flow": null,
4298
            "grid_area": null,
4299
            "grid_auto_columns": null,
4300
            "grid_auto_flow": null,
4301
            "grid_auto_rows": null,
4302
            "grid_column": null,
4303
            "grid_gap": null,
4304
            "grid_row": null,
4305
            "grid_template_areas": null,
4306
            "grid_template_columns": null,
4307
            "grid_template_rows": null,
4308
            "height": null,
4309
            "justify_content": null,
4310
            "justify_items": null,
4311
            "left": null,
4312
            "margin": null,
4313
            "max_height": null,
4314
            "max_width": null,
4315
            "min_height": null,
4316
            "min_width": null,
4317
            "object_fit": null,
4318
            "object_position": null,
4319
            "order": null,
4320
            "overflow": null,
4321
            "overflow_x": null,
4322
            "overflow_y": null,
4323
            "padding": null,
4324
            "right": null,
4325
            "top": null,
4326
            "visibility": null,
4327
            "width": null
4328
          }
4329
        },
4330
        "dd77325d1d8d417eb6544ae0cbc15133": {
4331
          "model_module": "@jupyter-widgets/controls",
4332
          "model_name": "DescriptionStyleModel",
4333
          "model_module_version": "1.5.0",
4334
          "state": {
4335
            "_model_module": "@jupyter-widgets/controls",
4336
            "_model_module_version": "1.5.0",
4337
            "_model_name": "DescriptionStyleModel",
4338
            "_view_count": null,
4339
            "_view_module": "@jupyter-widgets/base",
4340
            "_view_module_version": "1.2.0",
4341
            "_view_name": "StyleView",
4342
            "description_width": ""
4343
          }
4344
        },
4345
        "794c3466fdc34e549aaefbc106537e16": {
4346
          "model_module": "@jupyter-widgets/controls",
4347
          "model_name": "HBoxModel",
4348
          "model_module_version": "1.5.0",
4349
          "state": {
4350
            "_dom_classes": [],
4351
            "_model_module": "@jupyter-widgets/controls",
4352
            "_model_module_version": "1.5.0",
4353
            "_model_name": "HBoxModel",
4354
            "_view_count": null,
4355
            "_view_module": "@jupyter-widgets/controls",
4356
            "_view_module_version": "1.5.0",
4357
            "_view_name": "HBoxView",
4358
            "box_style": "",
4359
            "children": [
4360
              "IPY_MODEL_cb4a5c788e9d44818ff61e2551ed750e",
4361
              "IPY_MODEL_fec4fa9c00d6429791bceeccc053bc05",
4362
              "IPY_MODEL_fef1d4090bcf4d77895d45567bb1ff41"
4363
            ],
4364
            "layout": "IPY_MODEL_06a0a6459ff14187acac58481400bf05"
4365
          }
4366
        },
4367
        "cb4a5c788e9d44818ff61e2551ed750e": {
4368
          "model_module": "@jupyter-widgets/controls",
4369
          "model_name": "HTMLModel",
4370
          "model_module_version": "1.5.0",
4371
          "state": {
4372
            "_dom_classes": [],
4373
            "_model_module": "@jupyter-widgets/controls",
4374
            "_model_module_version": "1.5.0",
4375
            "_model_name": "HTMLModel",
4376
            "_view_count": null,
4377
            "_view_module": "@jupyter-widgets/controls",
4378
            "_view_module_version": "1.5.0",
4379
            "_view_name": "HTMLView",
4380
            "description": "",
4381
            "description_tooltip": null,
4382
            "layout": "IPY_MODEL_6caf5b09e0f44382807c41193a2963d3",
4383
            "placeholder": "​",
4384
            "style": "IPY_MODEL_6c95ad4f7ae24185a3fa9dbf9f8f68a2",
4385
            "value": "Downloading (…)former_sequential.py: 100%"
4386
          }
4387
        },
4388
        "fec4fa9c00d6429791bceeccc053bc05": {
4389
          "model_module": "@jupyter-widgets/controls",
4390
          "model_name": "FloatProgressModel",
4391
          "model_module_version": "1.5.0",
4392
          "state": {
4393
            "_dom_classes": [],
4394
            "_model_module": "@jupyter-widgets/controls",
4395
            "_model_module_version": "1.5.0",
4396
            "_model_name": "FloatProgressModel",
4397
            "_view_count": null,
4398
            "_view_module": "@jupyter-widgets/controls",
4399
            "_view_module_version": "1.5.0",
4400
            "_view_name": "ProgressView",
4401
            "bar_style": "success",
4402
            "description": "",
4403
            "description_tooltip": null,
4404
            "layout": "IPY_MODEL_69b2fdbe3a5e4682963987fb914afec5",
4405
            "max": 2235,
4406
            "min": 0,
4407
            "orientation": "horizontal",
4408
            "style": "IPY_MODEL_38e5964aa7a744439c7e5ad26c162ead",
4409
            "value": 2235
4410
          }
4411
        },
4412
        "fef1d4090bcf4d77895d45567bb1ff41": {
4413
          "model_module": "@jupyter-widgets/controls",
4414
          "model_name": "HTMLModel",
4415
          "model_module_version": "1.5.0",
4416
          "state": {
4417
            "_dom_classes": [],
4418
            "_model_module": "@jupyter-widgets/controls",
4419
            "_model_module_version": "1.5.0",
4420
            "_model_name": "HTMLModel",
4421
            "_view_count": null,
4422
            "_view_module": "@jupyter-widgets/controls",
4423
            "_view_module_version": "1.5.0",
4424
            "_view_name": "HTMLView",
4425
            "description": "",
4426
            "description_tooltip": null,
4427
            "layout": "IPY_MODEL_8f1d717eba6942b8aca16cbc49bc6a6f",
4428
            "placeholder": "​",
4429
            "style": "IPY_MODEL_0168b508956141269412e88990ee3d67",
4430
            "value": " 2.23k/2.23k [00:00&lt;00:00, 124kB/s]"
4431
          }
4432
        },
4433
        "06a0a6459ff14187acac58481400bf05": {
4434
          "model_module": "@jupyter-widgets/base",
4435
          "model_name": "LayoutModel",
4436
          "model_module_version": "1.2.0",
4437
          "state": {
4438
            "_model_module": "@jupyter-widgets/base",
4439
            "_model_module_version": "1.2.0",
4440
            "_model_name": "LayoutModel",
4441
            "_view_count": null,
4442
            "_view_module": "@jupyter-widgets/base",
4443
            "_view_module_version": "1.2.0",
4444
            "_view_name": "LayoutView",
4445
            "align_content": null,
4446
            "align_items": null,
4447
            "align_self": null,
4448
            "border": null,
4449
            "bottom": null,
4450
            "display": null,
4451
            "flex": null,
4452
            "flex_flow": null,
4453
            "grid_area": null,
4454
            "grid_auto_columns": null,
4455
            "grid_auto_flow": null,
4456
            "grid_auto_rows": null,
4457
            "grid_column": null,
4458
            "grid_gap": null,
4459
            "grid_row": null,
4460
            "grid_template_areas": null,
4461
            "grid_template_columns": null,
4462
            "grid_template_rows": null,
4463
            "height": null,
4464
            "justify_content": null,
4465
            "justify_items": null,
4466
            "left": null,
4467
            "margin": null,
4468
            "max_height": null,
4469
            "max_width": null,
4470
            "min_height": null,
4471
            "min_width": null,
4472
            "object_fit": null,
4473
            "object_position": null,
4474
            "order": null,
4475
            "overflow": null,
4476
            "overflow_x": null,
4477
            "overflow_y": null,
4478
            "padding": null,
4479
            "right": null,
4480
            "top": null,
4481
            "visibility": null,
4482
            "width": null
4483
          }
4484
        },
4485
        "6caf5b09e0f44382807c41193a2963d3": {
4486
          "model_module": "@jupyter-widgets/base",
4487
          "model_name": "LayoutModel",
4488
          "model_module_version": "1.2.0",
4489
          "state": {
4490
            "_model_module": "@jupyter-widgets/base",
4491
            "_model_module_version": "1.2.0",
4492
            "_model_name": "LayoutModel",
4493
            "_view_count": null,
4494
            "_view_module": "@jupyter-widgets/base",
4495
            "_view_module_version": "1.2.0",
4496
            "_view_name": "LayoutView",
4497
            "align_content": null,
4498
            "align_items": null,
4499
            "align_self": null,
4500
            "border": null,
4501
            "bottom": null,
4502
            "display": null,
4503
            "flex": null,
4504
            "flex_flow": null,
4505
            "grid_area": null,
4506
            "grid_auto_columns": null,
4507
            "grid_auto_flow": null,
4508
            "grid_auto_rows": null,
4509
            "grid_column": null,
4510
            "grid_gap": null,
4511
            "grid_row": null,
4512
            "grid_template_areas": null,
4513
            "grid_template_columns": null,
4514
            "grid_template_rows": null,
4515
            "height": null,
4516
            "justify_content": null,
4517
            "justify_items": null,
4518
            "left": null,
4519
            "margin": null,
4520
            "max_height": null,
4521
            "max_width": null,
4522
            "min_height": null,
4523
            "min_width": null,
4524
            "object_fit": null,
4525
            "object_position": null,
4526
            "order": null,
4527
            "overflow": null,
4528
            "overflow_x": null,
4529
            "overflow_y": null,
4530
            "padding": null,
4531
            "right": null,
4532
            "top": null,
4533
            "visibility": null,
4534
            "width": null
4535
          }
4536
        },
4537
        "6c95ad4f7ae24185a3fa9dbf9f8f68a2": {
4538
          "model_module": "@jupyter-widgets/controls",
4539
          "model_name": "DescriptionStyleModel",
4540
          "model_module_version": "1.5.0",
4541
          "state": {
4542
            "_model_module": "@jupyter-widgets/controls",
4543
            "_model_module_version": "1.5.0",
4544
            "_model_name": "DescriptionStyleModel",
4545
            "_view_count": null,
4546
            "_view_module": "@jupyter-widgets/base",
4547
            "_view_module_version": "1.2.0",
4548
            "_view_name": "StyleView",
4549
            "description_width": ""
4550
          }
4551
        },
4552
        "69b2fdbe3a5e4682963987fb914afec5": {
4553
          "model_module": "@jupyter-widgets/base",
4554
          "model_name": "LayoutModel",
4555
          "model_module_version": "1.2.0",
4556
          "state": {
4557
            "_model_module": "@jupyter-widgets/base",
4558
            "_model_module_version": "1.2.0",
4559
            "_model_name": "LayoutModel",
4560
            "_view_count": null,
4561
            "_view_module": "@jupyter-widgets/base",
4562
            "_view_module_version": "1.2.0",
4563
            "_view_name": "LayoutView",
4564
            "align_content": null,
4565
            "align_items": null,
4566
            "align_self": null,
4567
            "border": null,
4568
            "bottom": null,
4569
            "display": null,
4570
            "flex": null,
4571
            "flex_flow": null,
4572
            "grid_area": null,
4573
            "grid_auto_columns": null,
4574
            "grid_auto_flow": null,
4575
            "grid_auto_rows": null,
4576
            "grid_column": null,
4577
            "grid_gap": null,
4578
            "grid_row": null,
4579
            "grid_template_areas": null,
4580
            "grid_template_columns": null,
4581
            "grid_template_rows": null,
4582
            "height": null,
4583
            "justify_content": null,
4584
            "justify_items": null,
4585
            "left": null,
4586
            "margin": null,
4587
            "max_height": null,
4588
            "max_width": null,
4589
            "min_height": null,
4590
            "min_width": null,
4591
            "object_fit": null,
4592
            "object_position": null,
4593
            "order": null,
4594
            "overflow": null,
4595
            "overflow_x": null,
4596
            "overflow_y": null,
4597
            "padding": null,
4598
            "right": null,
4599
            "top": null,
4600
            "visibility": null,
4601
            "width": null
4602
          }
4603
        },
4604
        "38e5964aa7a744439c7e5ad26c162ead": {
4605
          "model_module": "@jupyter-widgets/controls",
4606
          "model_name": "ProgressStyleModel",
4607
          "model_module_version": "1.5.0",
4608
          "state": {
4609
            "_model_module": "@jupyter-widgets/controls",
4610
            "_model_module_version": "1.5.0",
4611
            "_model_name": "ProgressStyleModel",
4612
            "_view_count": null,
4613
            "_view_module": "@jupyter-widgets/base",
4614
            "_view_module_version": "1.2.0",
4615
            "_view_name": "StyleView",
4616
            "bar_color": null,
4617
            "description_width": ""
4618
          }
4619
        },
4620
        "8f1d717eba6942b8aca16cbc49bc6a6f": {
4621
          "model_module": "@jupyter-widgets/base",
4622
          "model_name": "LayoutModel",
4623
          "model_module_version": "1.2.0",
4624
          "state": {
4625
            "_model_module": "@jupyter-widgets/base",
4626
            "_model_module_version": "1.2.0",
4627
            "_model_name": "LayoutModel",
4628
            "_view_count": null,
4629
            "_view_module": "@jupyter-widgets/base",
4630
            "_view_module_version": "1.2.0",
4631
            "_view_name": "LayoutView",
4632
            "align_content": null,
4633
            "align_items": null,
4634
            "align_self": null,
4635
            "border": null,
4636
            "bottom": null,
4637
            "display": null,
4638
            "flex": null,
4639
            "flex_flow": null,
4640
            "grid_area": null,
4641
            "grid_auto_columns": null,
4642
            "grid_auto_flow": null,
4643
            "grid_auto_rows": null,
4644
            "grid_column": null,
4645
            "grid_gap": null,
4646
            "grid_row": null,
4647
            "grid_template_areas": null,
4648
            "grid_template_columns": null,
4649
            "grid_template_rows": null,
4650
            "height": null,
4651
            "justify_content": null,
4652
            "justify_items": null,
4653
            "left": null,
4654
            "margin": null,
4655
            "max_height": null,
4656
            "max_width": null,
4657
            "min_height": null,
4658
            "min_width": null,
4659
            "object_fit": null,
4660
            "object_position": null,
4661
            "order": null,
4662
            "overflow": null,
4663
            "overflow_x": null,
4664
            "overflow_y": null,
4665
            "padding": null,
4666
            "right": null,
4667
            "top": null,
4668
            "visibility": null,
4669
            "width": null
4670
          }
4671
        },
4672
        "0168b508956141269412e88990ee3d67": {
4673
          "model_module": "@jupyter-widgets/controls",
4674
          "model_name": "DescriptionStyleModel",
4675
          "model_module_version": "1.5.0",
4676
          "state": {
4677
            "_model_module": "@jupyter-widgets/controls",
4678
            "_model_module_version": "1.5.0",
4679
            "_model_name": "DescriptionStyleModel",
4680
            "_view_count": null,
4681
            "_view_module": "@jupyter-widgets/base",
4682
            "_view_module_version": "1.2.0",
4683
            "_view_name": "StyleView",
4684
            "description_width": ""
4685
          }
4686
        },
4687
        "e8b48f5b7e2b4d8d9407cb667092a3e8": {
4688
          "model_module": "@jupyter-widgets/controls",
4689
          "model_name": "HBoxModel",
4690
          "model_module_version": "1.5.0",
4691
          "state": {
4692
            "_dom_classes": [],
4693
            "_model_module": "@jupyter-widgets/controls",
4694
            "_model_module_version": "1.5.0",
4695
            "_model_name": "HBoxModel",
4696
            "_view_count": null,
4697
            "_view_module": "@jupyter-widgets/controls",
4698
            "_view_module_version": "1.5.0",
4699
            "_view_name": "HBoxView",
4700
            "box_style": "",
4701
            "children": [
4702
              "IPY_MODEL_8b1317e6869e439092c1d880137f62fd",
4703
              "IPY_MODEL_72a199e2dc1b4d63984493e6bf1bdb8c",
4704
              "IPY_MODEL_2296ca45af0c4cec80d01fb7e33f278f"
4705
            ],
4706
            "layout": "IPY_MODEL_3dd72eb426fa4e6e919276bb4e91f60f"
4707
          }
4708
        },
4709
        "8b1317e6869e439092c1d880137f62fd": {
4710
          "model_module": "@jupyter-widgets/controls",
4711
          "model_name": "HTMLModel",
4712
          "model_module_version": "1.5.0",
4713
          "state": {
4714
            "_dom_classes": [],
4715
            "_model_module": "@jupyter-widgets/controls",
4716
            "_model_module_version": "1.5.0",
4717
            "_model_name": "HTMLModel",
4718
            "_view_count": null,
4719
            "_view_module": "@jupyter-widgets/controls",
4720
            "_view_module_version": "1.5.0",
4721
            "_view_name": "HTMLView",
4722
            "description": "",
4723
            "description_tooltip": null,
4724
            "layout": "IPY_MODEL_2d01ed4a650c4bcf9d07f020ae98081d",
4725
            "placeholder": "​",
4726
            "style": "IPY_MODEL_51ac6641cd1441998abadc1993d4ef96",
4727
            "value": "Downloading (…)former_sequential.py: 100%"
4728
          }
4729
        },
4730
        "72a199e2dc1b4d63984493e6bf1bdb8c": {
4731
          "model_module": "@jupyter-widgets/controls",
4732
          "model_name": "FloatProgressModel",
4733
          "model_module_version": "1.5.0",
4734
          "state": {
4735
            "_dom_classes": [],
4736
            "_model_module": "@jupyter-widgets/controls",
4737
            "_model_module_version": "1.5.0",
4738
            "_model_name": "FloatProgressModel",
4739
            "_view_count": null,
4740
            "_view_module": "@jupyter-widgets/controls",
4741
            "_view_module_version": "1.5.0",
4742
            "_view_name": "ProgressView",
4743
            "bar_style": "success",
4744
            "description": "",
4745
            "description_tooltip": null,
4746
            "layout": "IPY_MODEL_520db9e6803d48fca60b1063e10cf260",
4747
            "max": 32240,
4748
            "min": 0,
4749
            "orientation": "horizontal",
4750
            "style": "IPY_MODEL_677f54e2735f407088724a38212c0171",
4751
            "value": 32240
4752
          }
4753
        },
4754
        "2296ca45af0c4cec80d01fb7e33f278f": {
4755
          "model_module": "@jupyter-widgets/controls",
4756
          "model_name": "HTMLModel",
4757
          "model_module_version": "1.5.0",
4758
          "state": {
4759
            "_dom_classes": [],
4760
            "_model_module": "@jupyter-widgets/controls",
4761
            "_model_module_version": "1.5.0",
4762
            "_model_name": "HTMLModel",
4763
            "_view_count": null,
4764
            "_view_module": "@jupyter-widgets/controls",
4765
            "_view_module_version": "1.5.0",
4766
            "_view_name": "HTMLView",
4767
            "description": "",
4768
            "description_tooltip": null,
4769
            "layout": "IPY_MODEL_49fd03a8e4954f5caf410898882324c8",
4770
            "placeholder": "​",
4771
            "style": "IPY_MODEL_63e68a09d85a4afd90d125bc61983d7a",
4772
            "value": " 32.2k/32.2k [00:00&lt;00:00, 951kB/s]"
4773
          }
4774
        },
4775
        "3dd72eb426fa4e6e919276bb4e91f60f": {
4776
          "model_module": "@jupyter-widgets/base",
4777
          "model_name": "LayoutModel",
4778
          "model_module_version": "1.2.0",
4779
          "state": {
4780
            "_model_module": "@jupyter-widgets/base",
4781
            "_model_module_version": "1.2.0",
4782
            "_model_name": "LayoutModel",
4783
            "_view_count": null,
4784
            "_view_module": "@jupyter-widgets/base",
4785
            "_view_module_version": "1.2.0",
4786
            "_view_name": "LayoutView",
4787
            "align_content": null,
4788
            "align_items": null,
4789
            "align_self": null,
4790
            "border": null,
4791
            "bottom": null,
4792
            "display": null,
4793
            "flex": null,
4794
            "flex_flow": null,
4795
            "grid_area": null,
4796
            "grid_auto_columns": null,
4797
            "grid_auto_flow": null,
4798
            "grid_auto_rows": null,
4799
            "grid_column": null,
4800
            "grid_gap": null,
4801
            "grid_row": null,
4802
            "grid_template_areas": null,
4803
            "grid_template_columns": null,
4804
            "grid_template_rows": null,
4805
            "height": null,
4806
            "justify_content": null,
4807
            "justify_items": null,
4808
            "left": null,
4809
            "margin": null,
4810
            "max_height": null,
4811
            "max_width": null,
4812
            "min_height": null,
4813
            "min_width": null,
4814
            "object_fit": null,
4815
            "object_position": null,
4816
            "order": null,
4817
            "overflow": null,
4818
            "overflow_x": null,
4819
            "overflow_y": null,
4820
            "padding": null,
4821
            "right": null,
4822
            "top": null,
4823
            "visibility": null,
4824
            "width": null
4825
          }
4826
        },
4827
        "2d01ed4a650c4bcf9d07f020ae98081d": {
4828
          "model_module": "@jupyter-widgets/base",
4829
          "model_name": "LayoutModel",
4830
          "model_module_version": "1.2.0",
4831
          "state": {
4832
            "_model_module": "@jupyter-widgets/base",
4833
            "_model_module_version": "1.2.0",
4834
            "_model_name": "LayoutModel",
4835
            "_view_count": null,
4836
            "_view_module": "@jupyter-widgets/base",
4837
            "_view_module_version": "1.2.0",
4838
            "_view_name": "LayoutView",
4839
            "align_content": null,
4840
            "align_items": null,
4841
            "align_self": null,
4842
            "border": null,
4843
            "bottom": null,
4844
            "display": null,
4845
            "flex": null,
4846
            "flex_flow": null,
4847
            "grid_area": null,
4848
            "grid_auto_columns": null,
4849
            "grid_auto_flow": null,
4850
            "grid_auto_rows": null,
4851
            "grid_column": null,
4852
            "grid_gap": null,
4853
            "grid_row": null,
4854
            "grid_template_areas": null,
4855
            "grid_template_columns": null,
4856
            "grid_template_rows": null,
4857
            "height": null,
4858
            "justify_content": null,
4859
            "justify_items": null,
4860
            "left": null,
4861
            "margin": null,
4862
            "max_height": null,
4863
            "max_width": null,
4864
            "min_height": null,
4865
            "min_width": null,
4866
            "object_fit": null,
4867
            "object_position": null,
4868
            "order": null,
4869
            "overflow": null,
4870
            "overflow_x": null,
4871
            "overflow_y": null,
4872
            "padding": null,
4873
            "right": null,
4874
            "top": null,
4875
            "visibility": null,
4876
            "width": null
4877
          }
4878
        },
4879
        "51ac6641cd1441998abadc1993d4ef96": {
4880
          "model_module": "@jupyter-widgets/controls",
4881
          "model_name": "DescriptionStyleModel",
4882
          "model_module_version": "1.5.0",
4883
          "state": {
4884
            "_model_module": "@jupyter-widgets/controls",
4885
            "_model_module_version": "1.5.0",
4886
            "_model_name": "DescriptionStyleModel",
4887
            "_view_count": null,
4888
            "_view_module": "@jupyter-widgets/base",
4889
            "_view_module_version": "1.2.0",
4890
            "_view_name": "StyleView",
4891
            "description_width": ""
4892
          }
4893
        },
4894
        "520db9e6803d48fca60b1063e10cf260": {
4895
          "model_module": "@jupyter-widgets/base",
4896
          "model_name": "LayoutModel",
4897
          "model_module_version": "1.2.0",
4898
          "state": {
4899
            "_model_module": "@jupyter-widgets/base",
4900
            "_model_module_version": "1.2.0",
4901
            "_model_name": "LayoutModel",
4902
            "_view_count": null,
4903
            "_view_module": "@jupyter-widgets/base",
4904
            "_view_module_version": "1.2.0",
4905
            "_view_name": "LayoutView",
4906
            "align_content": null,
4907
            "align_items": null,
4908
            "align_self": null,
4909
            "border": null,
4910
            "bottom": null,
4911
            "display": null,
4912
            "flex": null,
4913
            "flex_flow": null,
4914
            "grid_area": null,
4915
            "grid_auto_columns": null,
4916
            "grid_auto_flow": null,
4917
            "grid_auto_rows": null,
4918
            "grid_column": null,
4919
            "grid_gap": null,
4920
            "grid_row": null,
4921
            "grid_template_areas": null,
4922
            "grid_template_columns": null,
4923
            "grid_template_rows": null,
4924
            "height": null,
4925
            "justify_content": null,
4926
            "justify_items": null,
4927
            "left": null,
4928
            "margin": null,
4929
            "max_height": null,
4930
            "max_width": null,
4931
            "min_height": null,
4932
            "min_width": null,
4933
            "object_fit": null,
4934
            "object_position": null,
4935
            "order": null,
4936
            "overflow": null,
4937
            "overflow_x": null,
4938
            "overflow_y": null,
4939
            "padding": null,
4940
            "right": null,
4941
            "top": null,
4942
            "visibility": null,
4943
            "width": null
4944
          }
4945
        },
4946
        "677f54e2735f407088724a38212c0171": {
4947
          "model_module": "@jupyter-widgets/controls",
4948
          "model_name": "ProgressStyleModel",
4949
          "model_module_version": "1.5.0",
4950
          "state": {
4951
            "_model_module": "@jupyter-widgets/controls",
4952
            "_model_module_version": "1.5.0",
4953
            "_model_name": "ProgressStyleModel",
4954
            "_view_count": null,
4955
            "_view_module": "@jupyter-widgets/base",
4956
            "_view_module_version": "1.2.0",
4957
            "_view_name": "StyleView",
4958
            "bar_color": null,
4959
            "description_width": ""
4960
          }
4961
        },
4962
        "49fd03a8e4954f5caf410898882324c8": {
4963
          "model_module": "@jupyter-widgets/base",
4964
          "model_name": "LayoutModel",
4965
          "model_module_version": "1.2.0",
4966
          "state": {
4967
            "_model_module": "@jupyter-widgets/base",
4968
            "_model_module_version": "1.2.0",
4969
            "_model_name": "LayoutModel",
4970
            "_view_count": null,
4971
            "_view_module": "@jupyter-widgets/base",
4972
            "_view_module_version": "1.2.0",
4973
            "_view_name": "LayoutView",
4974
            "align_content": null,
4975
            "align_items": null,
4976
            "align_self": null,
4977
            "border": null,
4978
            "bottom": null,
4979
            "display": null,
4980
            "flex": null,
4981
            "flex_flow": null,
4982
            "grid_area": null,
4983
            "grid_auto_columns": null,
4984
            "grid_auto_flow": null,
4985
            "grid_auto_rows": null,
4986
            "grid_column": null,
4987
            "grid_gap": null,
4988
            "grid_row": null,
4989
            "grid_template_areas": null,
4990
            "grid_template_columns": null,
4991
            "grid_template_rows": null,
4992
            "height": null,
4993
            "justify_content": null,
4994
            "justify_items": null,
4995
            "left": null,
4996
            "margin": null,
4997
            "max_height": null,
4998
            "max_width": null,
4999
            "min_height": null,
5000
            "min_width": null,
5001
            "object_fit": null,
5002
            "object_position": null,
5003
            "order": null,
5004
            "overflow": null,
5005
            "overflow_x": null,
5006
            "overflow_y": null,
5007
            "padding": null,
5008
            "right": null,
5009
            "top": null,
5010
            "visibility": null,
5011
            "width": null
5012
          }
5013
        },
5014
        "63e68a09d85a4afd90d125bc61983d7a": {
5015
          "model_module": "@jupyter-widgets/controls",
5016
          "model_name": "DescriptionStyleModel",
5017
          "model_module_version": "1.5.0",
5018
          "state": {
5019
            "_model_module": "@jupyter-widgets/controls",
5020
            "_model_module_version": "1.5.0",
5021
            "_model_name": "DescriptionStyleModel",
5022
            "_view_count": null,
5023
            "_view_module": "@jupyter-widgets/base",
5024
            "_view_module_version": "1.2.0",
5025
            "_view_name": "StyleView",
5026
            "description_width": ""
5027
          }
5028
        },
5029
        "3672b47a27d64466995731f7475dff05": {
5030
          "model_module": "@jupyter-widgets/controls",
5031
          "model_name": "HBoxModel",
5032
          "model_module_version": "1.5.0",
5033
          "state": {
5034
            "_dom_classes": [],
5035
            "_model_module": "@jupyter-widgets/controls",
5036
            "_model_module_version": "1.5.0",
5037
            "_model_name": "HBoxModel",
5038
            "_view_count": null,
5039
            "_view_module": "@jupyter-widgets/controls",
5040
            "_view_module_version": "1.5.0",
5041
            "_view_name": "HBoxView",
5042
            "box_style": "",
5043
            "children": [
5044
              "IPY_MODEL_27905804572740018fa7c00a6de8e82d",
5045
              "IPY_MODEL_dd8375f821bd4283b3f906aea5c23d07",
5046
              "IPY_MODEL_a9f84e0028e440828def8cee6ae317ee"
5047
            ],
5048
            "layout": "IPY_MODEL_0c068bce19d74f7b835c90221a1cde38"
5049
          }
5050
        },
5051
        "27905804572740018fa7c00a6de8e82d": {
5052
          "model_module": "@jupyter-widgets/controls",
5053
          "model_name": "HTMLModel",
5054
          "model_module_version": "1.5.0",
5055
          "state": {
5056
            "_dom_classes": [],
5057
            "_model_module": "@jupyter-widgets/controls",
5058
            "_model_module_version": "1.5.0",
5059
            "_model_name": "HTMLModel",
5060
            "_view_count": null,
5061
            "_view_module": "@jupyter-widgets/controls",
5062
            "_view_module_version": "1.5.0",
5063
            "_view_name": "HTMLView",
5064
            "description": "",
5065
            "description_tooltip": null,
5066
            "layout": "IPY_MODEL_f9df8cdf043a4aaab82d55f24c17a3d8",
5067
            "placeholder": "​",
5068
            "style": "IPY_MODEL_4efaf38f2eaa46799c2afecb55ccf14c",
5069
            "value": "Downloading pytorch_model.bin: 100%"
5070
          }
5071
        },
5072
        "dd8375f821bd4283b3f906aea5c23d07": {
5073
          "model_module": "@jupyter-widgets/controls",
5074
          "model_name": "FloatProgressModel",
5075
          "model_module_version": "1.5.0",
5076
          "state": {
5077
            "_dom_classes": [],
5078
            "_model_module": "@jupyter-widgets/controls",
5079
            "_model_module_version": "1.5.0",
5080
            "_model_name": "FloatProgressModel",
5081
            "_view_count": null,
5082
            "_view_module": "@jupyter-widgets/controls",
5083
            "_view_module_version": "1.5.0",
5084
            "_view_name": "ProgressView",
5085
            "bar_style": "success",
5086
            "description": "",
5087
            "description_tooltip": null,
5088
            "layout": "IPY_MODEL_498424778ff94f7ea6e6f32e8cd95868",
5089
            "max": 2836623617,
5090
            "min": 0,
5091
            "orientation": "horizontal",
5092
            "style": "IPY_MODEL_6b654ce3193745208af00288d6f9d5f9",
5093
            "value": 2836623617
5094
          }
5095
        },
5096
        "a9f84e0028e440828def8cee6ae317ee": {
5097
          "model_module": "@jupyter-widgets/controls",
5098
          "model_name": "HTMLModel",
5099
          "model_module_version": "1.5.0",
5100
          "state": {
5101
            "_dom_classes": [],
5102
            "_model_module": "@jupyter-widgets/controls",
5103
            "_model_module_version": "1.5.0",
5104
            "_model_name": "HTMLModel",
5105
            "_view_count": null,
5106
            "_view_module": "@jupyter-widgets/controls",
5107
            "_view_module_version": "1.5.0",
5108
            "_view_name": "HTMLView",
5109
            "description": "",
5110
            "description_tooltip": null,
5111
            "layout": "IPY_MODEL_681653beca9b4a6591f7008e0ff5ac3b",
5112
            "placeholder": "​",
5113
            "style": "IPY_MODEL_997bd19e08ff48bc92a83487598be156",
5114
            "value": " 2.84G/2.84G [00:28&lt;00:00, 65.3MB/s]"
5115
          }
5116
        },
5117
        "0c068bce19d74f7b835c90221a1cde38": {
5118
          "model_module": "@jupyter-widgets/base",
5119
          "model_name": "LayoutModel",
5120
          "model_module_version": "1.2.0",
5121
          "state": {
5122
            "_model_module": "@jupyter-widgets/base",
5123
            "_model_module_version": "1.2.0",
5124
            "_model_name": "LayoutModel",
5125
            "_view_count": null,
5126
            "_view_module": "@jupyter-widgets/base",
5127
            "_view_module_version": "1.2.0",
5128
            "_view_name": "LayoutView",
5129
            "align_content": null,
5130
            "align_items": null,
5131
            "align_self": null,
5132
            "border": null,
5133
            "bottom": null,
5134
            "display": null,
5135
            "flex": null,
5136
            "flex_flow": null,
5137
            "grid_area": null,
5138
            "grid_auto_columns": null,
5139
            "grid_auto_flow": null,
5140
            "grid_auto_rows": null,
5141
            "grid_column": null,
5142
            "grid_gap": null,
5143
            "grid_row": null,
5144
            "grid_template_areas": null,
5145
            "grid_template_columns": null,
5146
            "grid_template_rows": null,
5147
            "height": null,
5148
            "justify_content": null,
5149
            "justify_items": null,
5150
            "left": null,
5151
            "margin": null,
5152
            "max_height": null,
5153
            "max_width": null,
5154
            "min_height": null,
5155
            "min_width": null,
5156
            "object_fit": null,
5157
            "object_position": null,
5158
            "order": null,
5159
            "overflow": null,
5160
            "overflow_x": null,
5161
            "overflow_y": null,
5162
            "padding": null,
5163
            "right": null,
5164
            "top": null,
5165
            "visibility": null,
5166
            "width": null
5167
          }
5168
        },
5169
        "f9df8cdf043a4aaab82d55f24c17a3d8": {
5170
          "model_module": "@jupyter-widgets/base",
5171
          "model_name": "LayoutModel",
5172
          "model_module_version": "1.2.0",
5173
          "state": {
5174
            "_model_module": "@jupyter-widgets/base",
5175
            "_model_module_version": "1.2.0",
5176
            "_model_name": "LayoutModel",
5177
            "_view_count": null,
5178
            "_view_module": "@jupyter-widgets/base",
5179
            "_view_module_version": "1.2.0",
5180
            "_view_name": "LayoutView",
5181
            "align_content": null,
5182
            "align_items": null,
5183
            "align_self": null,
5184
            "border": null,
5185
            "bottom": null,
5186
            "display": null,
5187
            "flex": null,
5188
            "flex_flow": null,
5189
            "grid_area": null,
5190
            "grid_auto_columns": null,
5191
            "grid_auto_flow": null,
5192
            "grid_auto_rows": null,
5193
            "grid_column": null,
5194
            "grid_gap": null,
5195
            "grid_row": null,
5196
            "grid_template_areas": null,
5197
            "grid_template_columns": null,
5198
            "grid_template_rows": null,
5199
            "height": null,
5200
            "justify_content": null,
5201
            "justify_items": null,
5202
            "left": null,
5203
            "margin": null,
5204
            "max_height": null,
5205
            "max_width": null,
5206
            "min_height": null,
5207
            "min_width": null,
5208
            "object_fit": null,
5209
            "object_position": null,
5210
            "order": null,
5211
            "overflow": null,
5212
            "overflow_x": null,
5213
            "overflow_y": null,
5214
            "padding": null,
5215
            "right": null,
5216
            "top": null,
5217
            "visibility": null,
5218
            "width": null
5219
          }
5220
        },
5221
        "4efaf38f2eaa46799c2afecb55ccf14c": {
5222
          "model_module": "@jupyter-widgets/controls",
5223
          "model_name": "DescriptionStyleModel",
5224
          "model_module_version": "1.5.0",
5225
          "state": {
5226
            "_model_module": "@jupyter-widgets/controls",
5227
            "_model_module_version": "1.5.0",
5228
            "_model_name": "DescriptionStyleModel",
5229
            "_view_count": null,
5230
            "_view_module": "@jupyter-widgets/base",
5231
            "_view_module_version": "1.2.0",
5232
            "_view_name": "StyleView",
5233
            "description_width": ""
5234
          }
5235
        },
5236
        "498424778ff94f7ea6e6f32e8cd95868": {
5237
          "model_module": "@jupyter-widgets/base",
5238
          "model_name": "LayoutModel",
5239
          "model_module_version": "1.2.0",
5240
          "state": {
5241
            "_model_module": "@jupyter-widgets/base",
5242
            "_model_module_version": "1.2.0",
5243
            "_model_name": "LayoutModel",
5244
            "_view_count": null,
5245
            "_view_module": "@jupyter-widgets/base",
5246
            "_view_module_version": "1.2.0",
5247
            "_view_name": "LayoutView",
5248
            "align_content": null,
5249
            "align_items": null,
5250
            "align_self": null,
5251
            "border": null,
5252
            "bottom": null,
5253
            "display": null,
5254
            "flex": null,
5255
            "flex_flow": null,
5256
            "grid_area": null,
5257
            "grid_auto_columns": null,
5258
            "grid_auto_flow": null,
5259
            "grid_auto_rows": null,
5260
            "grid_column": null,
5261
            "grid_gap": null,
5262
            "grid_row": null,
5263
            "grid_template_areas": null,
5264
            "grid_template_columns": null,
5265
            "grid_template_rows": null,
5266
            "height": null,
5267
            "justify_content": null,
5268
            "justify_items": null,
5269
            "left": null,
5270
            "margin": null,
5271
            "max_height": null,
5272
            "max_width": null,
5273
            "min_height": null,
5274
            "min_width": null,
5275
            "object_fit": null,
5276
            "object_position": null,
5277
            "order": null,
5278
            "overflow": null,
5279
            "overflow_x": null,
5280
            "overflow_y": null,
5281
            "padding": null,
5282
            "right": null,
5283
            "top": null,
5284
            "visibility": null,
5285
            "width": null
5286
          }
5287
        },
5288
        "6b654ce3193745208af00288d6f9d5f9": {
5289
          "model_module": "@jupyter-widgets/controls",
5290
          "model_name": "ProgressStyleModel",
5291
          "model_module_version": "1.5.0",
5292
          "state": {
5293
            "_model_module": "@jupyter-widgets/controls",
5294
            "_model_module_version": "1.5.0",
5295
            "_model_name": "ProgressStyleModel",
5296
            "_view_count": null,
5297
            "_view_module": "@jupyter-widgets/base",
5298
            "_view_module_version": "1.2.0",
5299
            "_view_name": "StyleView",
5300
            "bar_color": null,
5301
            "description_width": ""
5302
          }
5303
        },
5304
        "681653beca9b4a6591f7008e0ff5ac3b": {
5305
          "model_module": "@jupyter-widgets/base",
5306
          "model_name": "LayoutModel",
5307
          "model_module_version": "1.2.0",
5308
          "state": {
5309
            "_model_module": "@jupyter-widgets/base",
5310
            "_model_module_version": "1.2.0",
5311
            "_model_name": "LayoutModel",
5312
            "_view_count": null,
5313
            "_view_module": "@jupyter-widgets/base",
5314
            "_view_module_version": "1.2.0",
5315
            "_view_name": "LayoutView",
5316
            "align_content": null,
5317
            "align_items": null,
5318
            "align_self": null,
5319
            "border": null,
5320
            "bottom": null,
5321
            "display": null,
5322
            "flex": null,
5323
            "flex_flow": null,
5324
            "grid_area": null,
5325
            "grid_auto_columns": null,
5326
            "grid_auto_flow": null,
5327
            "grid_auto_rows": null,
5328
            "grid_column": null,
5329
            "grid_gap": null,
5330
            "grid_row": null,
5331
            "grid_template_areas": null,
5332
            "grid_template_columns": null,
5333
            "grid_template_rows": null,
5334
            "height": null,
5335
            "justify_content": null,
5336
            "justify_items": null,
5337
            "left": null,
5338
            "margin": null,
5339
            "max_height": null,
5340
            "max_width": null,
5341
            "min_height": null,
5342
            "min_width": null,
5343
            "object_fit": null,
5344
            "object_position": null,
5345
            "order": null,
5346
            "overflow": null,
5347
            "overflow_x": null,
5348
            "overflow_y": null,
5349
            "padding": null,
5350
            "right": null,
5351
            "top": null,
5352
            "visibility": null,
5353
            "width": null
5354
          }
5355
        },
5356
        "997bd19e08ff48bc92a83487598be156": {
5357
          "model_module": "@jupyter-widgets/controls",
5358
          "model_name": "DescriptionStyleModel",
5359
          "model_module_version": "1.5.0",
5360
          "state": {
5361
            "_model_module": "@jupyter-widgets/controls",
5362
            "_model_module_version": "1.5.0",
5363
            "_model_name": "DescriptionStyleModel",
5364
            "_view_count": null,
5365
            "_view_module": "@jupyter-widgets/base",
5366
            "_view_module_version": "1.2.0",
5367
            "_view_name": "StyleView",
5368
            "description_width": ""
5369
          }
5370
        },
5371
        "27e56e4591da430fb5f62b5d49df7e9e": {
5372
          "model_module": "@jupyter-widgets/controls",
5373
          "model_name": "HBoxModel",
5374
          "model_module_version": "1.5.0",
5375
          "state": {
5376
            "_dom_classes": [],
5377
            "_model_module": "@jupyter-widgets/controls",
5378
            "_model_module_version": "1.5.0",
5379
            "_model_name": "HBoxModel",
5380
            "_view_count": null,
5381
            "_view_module": "@jupyter-widgets/controls",
5382
            "_view_module_version": "1.5.0",
5383
            "_view_name": "HBoxView",
5384
            "box_style": "",
5385
            "children": [
5386
              "IPY_MODEL_b90e6e929c164e3384cb0067555f66a4",
5387
              "IPY_MODEL_5fb49e2af10d452c9558f6f2b7551b16",
5388
              "IPY_MODEL_a6547c497b5b44b59c1de056965de351"
5389
            ],
5390
            "layout": "IPY_MODEL_8732438e1da948e6a4f74cde7a25eb57"
5391
          }
5392
        },
5393
        "b90e6e929c164e3384cb0067555f66a4": {
5394
          "model_module": "@jupyter-widgets/controls",
5395
          "model_name": "HTMLModel",
5396
          "model_module_version": "1.5.0",
5397
          "state": {
5398
            "_dom_classes": [],
5399
            "_model_module": "@jupyter-widgets/controls",
5400
            "_model_module_version": "1.5.0",
5401
            "_model_name": "HTMLModel",
5402
            "_view_count": null,
5403
            "_view_module": "@jupyter-widgets/controls",
5404
            "_view_module_version": "1.5.0",
5405
            "_view_name": "HTMLView",
5406
            "description": "",
5407
            "description_tooltip": null,
5408
            "layout": "IPY_MODEL_cdd90757935f45668a81520522e3f057",
5409
            "placeholder": "​",
5410
            "style": "IPY_MODEL_aa2269ddef7f4e66a42973e5fd3e568f",
5411
            "value": "Downloading (…)neration_config.json: 100%"
5412
          }
5413
        },
5414
        "5fb49e2af10d452c9558f6f2b7551b16": {
5415
          "model_module": "@jupyter-widgets/controls",
5416
          "model_name": "FloatProgressModel",
5417
          "model_module_version": "1.5.0",
5418
          "state": {
5419
            "_dom_classes": [],
5420
            "_model_module": "@jupyter-widgets/controls",
5421
            "_model_module_version": "1.5.0",
5422
            "_model_name": "FloatProgressModel",
5423
            "_view_count": null,
5424
            "_view_module": "@jupyter-widgets/controls",
5425
            "_view_module_version": "1.5.0",
5426
            "_view_name": "ProgressView",
5427
            "bar_style": "success",
5428
            "description": "",
5429
            "description_tooltip": null,
5430
            "layout": "IPY_MODEL_a9d745f127214a4289bf880d65899dd3",
5431
            "max": 69,
5432
            "min": 0,
5433
            "orientation": "horizontal",
5434
            "style": "IPY_MODEL_90884fd8a8d24859b4f94aac4041cfe4",
5435
            "value": 69
5436
          }
5437
        },
5438
        "a6547c497b5b44b59c1de056965de351": {
5439
          "model_module": "@jupyter-widgets/controls",
5440
          "model_name": "HTMLModel",
5441
          "model_module_version": "1.5.0",
5442
          "state": {
5443
            "_dom_classes": [],
5444
            "_model_module": "@jupyter-widgets/controls",
5445
            "_model_module_version": "1.5.0",
5446
            "_model_name": "HTMLModel",
5447
            "_view_count": null,
5448
            "_view_module": "@jupyter-widgets/controls",
5449
            "_view_module_version": "1.5.0",
5450
            "_view_name": "HTMLView",
5451
            "description": "",
5452
            "description_tooltip": null,
5453
            "layout": "IPY_MODEL_0fced73c513242fabd2fc4eb30df1e21",
5454
            "placeholder": "​",
5455
            "style": "IPY_MODEL_22d1d55d6580458fb8fdb4af38ec4690",
5456
            "value": " 69.0/69.0 [00:00&lt;00:00, 4.76kB/s]"
5457
          }
5458
        },
5459
        "8732438e1da948e6a4f74cde7a25eb57": {
5460
          "model_module": "@jupyter-widgets/base",
5461
          "model_name": "LayoutModel",
5462
          "model_module_version": "1.2.0",
5463
          "state": {
5464
            "_model_module": "@jupyter-widgets/base",
5465
            "_model_module_version": "1.2.0",
5466
            "_model_name": "LayoutModel",
5467
            "_view_count": null,
5468
            "_view_module": "@jupyter-widgets/base",
5469
            "_view_module_version": "1.2.0",
5470
            "_view_name": "LayoutView",
5471
            "align_content": null,
5472
            "align_items": null,
5473
            "align_self": null,
5474
            "border": null,
5475
            "bottom": null,
5476
            "display": null,
5477
            "flex": null,
5478
            "flex_flow": null,
5479
            "grid_area": null,
5480
            "grid_auto_columns": null,
5481
            "grid_auto_flow": null,
5482
            "grid_auto_rows": null,
5483
            "grid_column": null,
5484
            "grid_gap": null,
5485
            "grid_row": null,
5486
            "grid_template_areas": null,
5487
            "grid_template_columns": null,
5488
            "grid_template_rows": null,
5489
            "height": null,
5490
            "justify_content": null,
5491
            "justify_items": null,
5492
            "left": null,
5493
            "margin": null,
5494
            "max_height": null,
5495
            "max_width": null,
5496
            "min_height": null,
5497
            "min_width": null,
5498
            "object_fit": null,
5499
            "object_position": null,
5500
            "order": null,
5501
            "overflow": null,
5502
            "overflow_x": null,
5503
            "overflow_y": null,
5504
            "padding": null,
5505
            "right": null,
5506
            "top": null,
5507
            "visibility": null,
5508
            "width": null
5509
          }
5510
        },
5511
        "cdd90757935f45668a81520522e3f057": {
5512
          "model_module": "@jupyter-widgets/base",
5513
          "model_name": "LayoutModel",
5514
          "model_module_version": "1.2.0",
5515
          "state": {
5516
            "_model_module": "@jupyter-widgets/base",
5517
            "_model_module_version": "1.2.0",
5518
            "_model_name": "LayoutModel",
5519
            "_view_count": null,
5520
            "_view_module": "@jupyter-widgets/base",
5521
            "_view_module_version": "1.2.0",
5522
            "_view_name": "LayoutView",
5523
            "align_content": null,
5524
            "align_items": null,
5525
            "align_self": null,
5526
            "border": null,
5527
            "bottom": null,
5528
            "display": null,
5529
            "flex": null,
5530
            "flex_flow": null,
5531
            "grid_area": null,
5532
            "grid_auto_columns": null,
5533
            "grid_auto_flow": null,
5534
            "grid_auto_rows": null,
5535
            "grid_column": null,
5536
            "grid_gap": null,
5537
            "grid_row": null,
5538
            "grid_template_areas": null,
5539
            "grid_template_columns": null,
5540
            "grid_template_rows": null,
5541
            "height": null,
5542
            "justify_content": null,
5543
            "justify_items": null,
5544
            "left": null,
5545
            "margin": null,
5546
            "max_height": null,
5547
            "max_width": null,
5548
            "min_height": null,
5549
            "min_width": null,
5550
            "object_fit": null,
5551
            "object_position": null,
5552
            "order": null,
5553
            "overflow": null,
5554
            "overflow_x": null,
5555
            "overflow_y": null,
5556
            "padding": null,
5557
            "right": null,
5558
            "top": null,
5559
            "visibility": null,
5560
            "width": null
5561
          }
5562
        },
5563
        "aa2269ddef7f4e66a42973e5fd3e568f": {
5564
          "model_module": "@jupyter-widgets/controls",
5565
          "model_name": "DescriptionStyleModel",
5566
          "model_module_version": "1.5.0",
5567
          "state": {
5568
            "_model_module": "@jupyter-widgets/controls",
5569
            "_model_module_version": "1.5.0",
5570
            "_model_name": "DescriptionStyleModel",
5571
            "_view_count": null,
5572
            "_view_module": "@jupyter-widgets/base",
5573
            "_view_module_version": "1.2.0",
5574
            "_view_name": "StyleView",
5575
            "description_width": ""
5576
          }
5577
        },
5578
        "a9d745f127214a4289bf880d65899dd3": {
5579
          "model_module": "@jupyter-widgets/base",
5580
          "model_name": "LayoutModel",
5581
          "model_module_version": "1.2.0",
5582
          "state": {
5583
            "_model_module": "@jupyter-widgets/base",
5584
            "_model_module_version": "1.2.0",
5585
            "_model_name": "LayoutModel",
5586
            "_view_count": null,
5587
            "_view_module": "@jupyter-widgets/base",
5588
            "_view_module_version": "1.2.0",
5589
            "_view_name": "LayoutView",
5590
            "align_content": null,
5591
            "align_items": null,
5592
            "align_self": null,
5593
            "border": null,
5594
            "bottom": null,
5595
            "display": null,
5596
            "flex": null,
5597
            "flex_flow": null,
5598
            "grid_area": null,
5599
            "grid_auto_columns": null,
5600
            "grid_auto_flow": null,
5601
            "grid_auto_rows": null,
5602
            "grid_column": null,
5603
            "grid_gap": null,
5604
            "grid_row": null,
5605
            "grid_template_areas": null,
5606
            "grid_template_columns": null,
5607
            "grid_template_rows": null,
5608
            "height": null,
5609
            "justify_content": null,
5610
            "justify_items": null,
5611
            "left": null,
5612
            "margin": null,
5613
            "max_height": null,
5614
            "max_width": null,
5615
            "min_height": null,
5616
            "min_width": null,
5617
            "object_fit": null,
5618
            "object_position": null,
5619
            "order": null,
5620
            "overflow": null,
5621
            "overflow_x": null,
5622
            "overflow_y": null,
5623
            "padding": null,
5624
            "right": null,
5625
            "top": null,
5626
            "visibility": null,
5627
            "width": null
5628
          }
5629
        },
5630
        "90884fd8a8d24859b4f94aac4041cfe4": {
5631
          "model_module": "@jupyter-widgets/controls",
5632
          "model_name": "ProgressStyleModel",
5633
          "model_module_version": "1.5.0",
5634
          "state": {
5635
            "_model_module": "@jupyter-widgets/controls",
5636
            "_model_module_version": "1.5.0",
5637
            "_model_name": "ProgressStyleModel",
5638
            "_view_count": null,
5639
            "_view_module": "@jupyter-widgets/base",
5640
            "_view_module_version": "1.2.0",
5641
            "_view_name": "StyleView",
5642
            "bar_color": null,
5643
            "description_width": ""
5644
          }
5645
        },
5646
        "0fced73c513242fabd2fc4eb30df1e21": {
5647
          "model_module": "@jupyter-widgets/base",
5648
          "model_name": "LayoutModel",
5649
          "model_module_version": "1.2.0",
5650
          "state": {
5651
            "_model_module": "@jupyter-widgets/base",
5652
            "_model_module_version": "1.2.0",
5653
            "_model_name": "LayoutModel",
5654
            "_view_count": null,
5655
            "_view_module": "@jupyter-widgets/base",
5656
            "_view_module_version": "1.2.0",
5657
            "_view_name": "LayoutView",
5658
            "align_content": null,
5659
            "align_items": null,
5660
            "align_self": null,
5661
            "border": null,
5662
            "bottom": null,
5663
            "display": null,
5664
            "flex": null,
5665
            "flex_flow": null,
5666
            "grid_area": null,
5667
            "grid_auto_columns": null,
5668
            "grid_auto_flow": null,
5669
            "grid_auto_rows": null,
5670
            "grid_column": null,
5671
            "grid_gap": null,
5672
            "grid_row": null,
5673
            "grid_template_areas": null,
5674
            "grid_template_columns": null,
5675
            "grid_template_rows": null,
5676
            "height": null,
5677
            "justify_content": null,
5678
            "justify_items": null,
5679
            "left": null,
5680
            "margin": null,
5681
            "max_height": null,
5682
            "max_width": null,
5683
            "min_height": null,
5684
            "min_width": null,
5685
            "object_fit": null,
5686
            "object_position": null,
5687
            "order": null,
5688
            "overflow": null,
5689
            "overflow_x": null,
5690
            "overflow_y": null,
5691
            "padding": null,
5692
            "right": null,
5693
            "top": null,
5694
            "visibility": null,
5695
            "width": null
5696
          }
5697
        },
5698
        "22d1d55d6580458fb8fdb4af38ec4690": {
5699
          "model_module": "@jupyter-widgets/controls",
5700
          "model_name": "DescriptionStyleModel",
5701
          "model_module_version": "1.5.0",
5702
          "state": {
5703
            "_model_module": "@jupyter-widgets/controls",
5704
            "_model_module_version": "1.5.0",
5705
            "_model_name": "DescriptionStyleModel",
5706
            "_view_count": null,
5707
            "_view_module": "@jupyter-widgets/base",
5708
            "_view_module_version": "1.2.0",
5709
            "_view_name": "StyleView",
5710
            "description_width": ""
5711
          }
5712
        },
5713
        "a4acb6dc90144e729a615bb9b6e2db55": {
5714
          "model_module": "@jupyter-widgets/controls",
5715
          "model_name": "HBoxModel",
5716
          "model_module_version": "1.5.0",
5717
          "state": {
5718
            "_dom_classes": [],
5719
            "_model_module": "@jupyter-widgets/controls",
5720
            "_model_module_version": "1.5.0",
5721
            "_model_name": "HBoxModel",
5722
            "_view_count": null,
5723
            "_view_module": "@jupyter-widgets/controls",
5724
            "_view_module_version": "1.5.0",
5725
            "_view_name": "HBoxView",
5726
            "box_style": "",
5727
            "children": [
5728
              "IPY_MODEL_3bc8f617a46e47fb8676ffee9cc25fba",
5729
              "IPY_MODEL_fe82a3b74c5b4f79aa367fe5d8f79345",
5730
              "IPY_MODEL_387d62eb068e44619908b006079eacd7"
5731
            ],
5732
            "layout": "IPY_MODEL_e6d16b50198a450c876640e34d17140e"
5733
          }
5734
        },
5735
        "3bc8f617a46e47fb8676ffee9cc25fba": {
5736
          "model_module": "@jupyter-widgets/controls",
5737
          "model_name": "HTMLModel",
5738
          "model_module_version": "1.5.0",
5739
          "state": {
5740
            "_dom_classes": [],
5741
            "_model_module": "@jupyter-widgets/controls",
5742
            "_model_module_version": "1.5.0",
5743
            "_model_name": "HTMLModel",
5744
            "_view_count": null,
5745
            "_view_module": "@jupyter-widgets/controls",
5746
            "_view_module_version": "1.5.0",
5747
            "_view_name": "HTMLView",
5748
            "description": "",
5749
            "description_tooltip": null,
5750
            "layout": "IPY_MODEL_b4a34fe788034852876cc1886c28042f",
5751
            "placeholder": "​",
5752
            "style": "IPY_MODEL_265d6ad184a74be9b9e8d7378bb98f8d",
5753
            "value": "Downloading (…)okenizer_config.json: 100%"
5754
          }
5755
        },
5756
        "fe82a3b74c5b4f79aa367fe5d8f79345": {
5757
          "model_module": "@jupyter-widgets/controls",
5758
          "model_name": "FloatProgressModel",
5759
          "model_module_version": "1.5.0",
5760
          "state": {
5761
            "_dom_classes": [],
5762
            "_model_module": "@jupyter-widgets/controls",
5763
            "_model_module_version": "1.5.0",
5764
            "_model_name": "FloatProgressModel",
5765
            "_view_count": null,
5766
            "_view_module": "@jupyter-widgets/controls",
5767
            "_view_module_version": "1.5.0",
5768
            "_view_name": "ProgressView",
5769
            "bar_style": "success",
5770
            "description": "",
5771
            "description_tooltip": null,
5772
            "layout": "IPY_MODEL_e25b91678ed246cd997cec87e918c079",
5773
            "max": 237,
5774
            "min": 0,
5775
            "orientation": "horizontal",
5776
            "style": "IPY_MODEL_7bf39b13ce4148ca8fe508c73b63e413",
5777
            "value": 237
5778
          }
5779
        },
5780
        "387d62eb068e44619908b006079eacd7": {
5781
          "model_module": "@jupyter-widgets/controls",
5782
          "model_name": "HTMLModel",
5783
          "model_module_version": "1.5.0",
5784
          "state": {
5785
            "_dom_classes": [],
5786
            "_model_module": "@jupyter-widgets/controls",
5787
            "_model_module_version": "1.5.0",
5788
            "_model_name": "HTMLModel",
5789
            "_view_count": null,
5790
            "_view_module": "@jupyter-widgets/controls",
5791
            "_view_module_version": "1.5.0",
5792
            "_view_name": "HTMLView",
5793
            "description": "",
5794
            "description_tooltip": null,
5795
            "layout": "IPY_MODEL_b0e8f79bf6b74f08801f385b4c7aadee",
5796
            "placeholder": "​",
5797
            "style": "IPY_MODEL_8a8d6a4614e14aaeb03e4bfa9b8a0741",
5798
            "value": " 237/237 [00:00&lt;00:00, 13.1kB/s]"
5799
          }
5800
        },
5801
        "e6d16b50198a450c876640e34d17140e": {
5802
          "model_module": "@jupyter-widgets/base",
5803
          "model_name": "LayoutModel",
5804
          "model_module_version": "1.2.0",
5805
          "state": {
5806
            "_model_module": "@jupyter-widgets/base",
5807
            "_model_module_version": "1.2.0",
5808
            "_model_name": "LayoutModel",
5809
            "_view_count": null,
5810
            "_view_module": "@jupyter-widgets/base",
5811
            "_view_module_version": "1.2.0",
5812
            "_view_name": "LayoutView",
5813
            "align_content": null,
5814
            "align_items": null,
5815
            "align_self": null,
5816
            "border": null,
5817
            "bottom": null,
5818
            "display": null,
5819
            "flex": null,
5820
            "flex_flow": null,
5821
            "grid_area": null,
5822
            "grid_auto_columns": null,
5823
            "grid_auto_flow": null,
5824
            "grid_auto_rows": null,
5825
            "grid_column": null,
5826
            "grid_gap": null,
5827
            "grid_row": null,
5828
            "grid_template_areas": null,
5829
            "grid_template_columns": null,
5830
            "grid_template_rows": null,
5831
            "height": null,
5832
            "justify_content": null,
5833
            "justify_items": null,
5834
            "left": null,
5835
            "margin": null,
5836
            "max_height": null,
5837
            "max_width": null,
5838
            "min_height": null,
5839
            "min_width": null,
5840
            "object_fit": null,
5841
            "object_position": null,
5842
            "order": null,
5843
            "overflow": null,
5844
            "overflow_x": null,
5845
            "overflow_y": null,
5846
            "padding": null,
5847
            "right": null,
5848
            "top": null,
5849
            "visibility": null,
5850
            "width": null
5851
          }
5852
        },
5853
        "b4a34fe788034852876cc1886c28042f": {
5854
          "model_module": "@jupyter-widgets/base",
5855
          "model_name": "LayoutModel",
5856
          "model_module_version": "1.2.0",
5857
          "state": {
5858
            "_model_module": "@jupyter-widgets/base",
5859
            "_model_module_version": "1.2.0",
5860
            "_model_name": "LayoutModel",
5861
            "_view_count": null,
5862
            "_view_module": "@jupyter-widgets/base",
5863
            "_view_module_version": "1.2.0",
5864
            "_view_name": "LayoutView",
5865
            "align_content": null,
5866
            "align_items": null,
5867
            "align_self": null,
5868
            "border": null,
5869
            "bottom": null,
5870
            "display": null,
5871
            "flex": null,
5872
            "flex_flow": null,
5873
            "grid_area": null,
5874
            "grid_auto_columns": null,
5875
            "grid_auto_flow": null,
5876
            "grid_auto_rows": null,
5877
            "grid_column": null,
5878
            "grid_gap": null,
5879
            "grid_row": null,
5880
            "grid_template_areas": null,
5881
            "grid_template_columns": null,
5882
            "grid_template_rows": null,
5883
            "height": null,
5884
            "justify_content": null,
5885
            "justify_items": null,
5886
            "left": null,
5887
            "margin": null,
5888
            "max_height": null,
5889
            "max_width": null,
5890
            "min_height": null,
5891
            "min_width": null,
5892
            "object_fit": null,
5893
            "object_position": null,
5894
            "order": null,
5895
            "overflow": null,
5896
            "overflow_x": null,
5897
            "overflow_y": null,
5898
            "padding": null,
5899
            "right": null,
5900
            "top": null,
5901
            "visibility": null,
5902
            "width": null
5903
          }
5904
        },
5905
        "265d6ad184a74be9b9e8d7378bb98f8d": {
5906
          "model_module": "@jupyter-widgets/controls",
5907
          "model_name": "DescriptionStyleModel",
5908
          "model_module_version": "1.5.0",
5909
          "state": {
5910
            "_model_module": "@jupyter-widgets/controls",
5911
            "_model_module_version": "1.5.0",
5912
            "_model_name": "DescriptionStyleModel",
5913
            "_view_count": null,
5914
            "_view_module": "@jupyter-widgets/base",
5915
            "_view_module_version": "1.2.0",
5916
            "_view_name": "StyleView",
5917
            "description_width": ""
5918
          }
5919
        },
5920
        "e25b91678ed246cd997cec87e918c079": {
5921
          "model_module": "@jupyter-widgets/base",
5922
          "model_name": "LayoutModel",
5923
          "model_module_version": "1.2.0",
5924
          "state": {
5925
            "_model_module": "@jupyter-widgets/base",
5926
            "_model_module_version": "1.2.0",
5927
            "_model_name": "LayoutModel",
5928
            "_view_count": null,
5929
            "_view_module": "@jupyter-widgets/base",
5930
            "_view_module_version": "1.2.0",
5931
            "_view_name": "LayoutView",
5932
            "align_content": null,
5933
            "align_items": null,
5934
            "align_self": null,
5935
            "border": null,
5936
            "bottom": null,
5937
            "display": null,
5938
            "flex": null,
5939
            "flex_flow": null,
5940
            "grid_area": null,
5941
            "grid_auto_columns": null,
5942
            "grid_auto_flow": null,
5943
            "grid_auto_rows": null,
5944
            "grid_column": null,
5945
            "grid_gap": null,
5946
            "grid_row": null,
5947
            "grid_template_areas": null,
5948
            "grid_template_columns": null,
5949
            "grid_template_rows": null,
5950
            "height": null,
5951
            "justify_content": null,
5952
            "justify_items": null,
5953
            "left": null,
5954
            "margin": null,
5955
            "max_height": null,
5956
            "max_width": null,
5957
            "min_height": null,
5958
            "min_width": null,
5959
            "object_fit": null,
5960
            "object_position": null,
5961
            "order": null,
5962
            "overflow": null,
5963
            "overflow_x": null,
5964
            "overflow_y": null,
5965
            "padding": null,
5966
            "right": null,
5967
            "top": null,
5968
            "visibility": null,
5969
            "width": null
5970
          }
5971
        },
5972
        "7bf39b13ce4148ca8fe508c73b63e413": {
5973
          "model_module": "@jupyter-widgets/controls",
5974
          "model_name": "ProgressStyleModel",
5975
          "model_module_version": "1.5.0",
5976
          "state": {
5977
            "_model_module": "@jupyter-widgets/controls",
5978
            "_model_module_version": "1.5.0",
5979
            "_model_name": "ProgressStyleModel",
5980
            "_view_count": null,
5981
            "_view_module": "@jupyter-widgets/base",
5982
            "_view_module_version": "1.2.0",
5983
            "_view_name": "StyleView",
5984
            "bar_color": null,
5985
            "description_width": ""
5986
          }
5987
        },
5988
        "b0e8f79bf6b74f08801f385b4c7aadee": {
5989
          "model_module": "@jupyter-widgets/base",
5990
          "model_name": "LayoutModel",
5991
          "model_module_version": "1.2.0",
5992
          "state": {
5993
            "_model_module": "@jupyter-widgets/base",
5994
            "_model_module_version": "1.2.0",
5995
            "_model_name": "LayoutModel",
5996
            "_view_count": null,
5997
            "_view_module": "@jupyter-widgets/base",
5998
            "_view_module_version": "1.2.0",
5999
            "_view_name": "LayoutView",
6000
            "align_content": null,
6001
            "align_items": null,
6002
            "align_self": null,
6003
            "border": null,
6004
            "bottom": null,
6005
            "display": null,
6006
            "flex": null,
6007
            "flex_flow": null,
6008
            "grid_area": null,
6009
            "grid_auto_columns": null,
6010
            "grid_auto_flow": null,
6011
            "grid_auto_rows": null,
6012
            "grid_column": null,
6013
            "grid_gap": null,
6014
            "grid_row": null,
6015
            "grid_template_areas": null,
6016
            "grid_template_columns": null,
6017
            "grid_template_rows": null,
6018
            "height": null,
6019
            "justify_content": null,
6020
            "justify_items": null,
6021
            "left": null,
6022
            "margin": null,
6023
            "max_height": null,
6024
            "max_width": null,
6025
            "min_height": null,
6026
            "min_width": null,
6027
            "object_fit": null,
6028
            "object_position": null,
6029
            "order": null,
6030
            "overflow": null,
6031
            "overflow_x": null,
6032
            "overflow_y": null,
6033
            "padding": null,
6034
            "right": null,
6035
            "top": null,
6036
            "visibility": null,
6037
            "width": null
6038
          }
6039
        },
6040
        "8a8d6a4614e14aaeb03e4bfa9b8a0741": {
6041
          "model_module": "@jupyter-widgets/controls",
6042
          "model_name": "DescriptionStyleModel",
6043
          "model_module_version": "1.5.0",
6044
          "state": {
6045
            "_model_module": "@jupyter-widgets/controls",
6046
            "_model_module_version": "1.5.0",
6047
            "_model_name": "DescriptionStyleModel",
6048
            "_view_count": null,
6049
            "_view_module": "@jupyter-widgets/base",
6050
            "_view_module_version": "1.2.0",
6051
            "_view_name": "StyleView",
6052
            "description_width": ""
6053
          }
6054
        },
6055
        "f14e8afa14b74002af2be74b84d8b033": {
6056
          "model_module": "@jupyter-widgets/controls",
6057
          "model_name": "HBoxModel",
6058
          "model_module_version": "1.5.0",
6059
          "state": {
6060
            "_dom_classes": [],
6061
            "_model_module": "@jupyter-widgets/controls",
6062
            "_model_module_version": "1.5.0",
6063
            "_model_name": "HBoxModel",
6064
            "_view_count": null,
6065
            "_view_module": "@jupyter-widgets/controls",
6066
            "_view_module_version": "1.5.0",
6067
            "_view_name": "HBoxView",
6068
            "box_style": "",
6069
            "children": [
6070
              "IPY_MODEL_17cc56f07af34e588c3d3e0cdba0b458",
6071
              "IPY_MODEL_9439477ef5b64c1ba5c517d76c2c460f",
6072
              "IPY_MODEL_fc8787cef48c4befae588efa56f99adc"
6073
            ],
6074
            "layout": "IPY_MODEL_fd97aec74a1148b0b94de3f2194a0558"
6075
          }
6076
        },
6077
        "17cc56f07af34e588c3d3e0cdba0b458": {
6078
          "model_module": "@jupyter-widgets/controls",
6079
          "model_name": "HTMLModel",
6080
          "model_module_version": "1.5.0",
6081
          "state": {
6082
            "_dom_classes": [],
6083
            "_model_module": "@jupyter-widgets/controls",
6084
            "_model_module_version": "1.5.0",
6085
            "_model_name": "HTMLModel",
6086
            "_view_count": null,
6087
            "_view_module": "@jupyter-widgets/controls",
6088
            "_view_module_version": "1.5.0",
6089
            "_view_name": "HTMLView",
6090
            "description": "",
6091
            "description_tooltip": null,
6092
            "layout": "IPY_MODEL_cf54b45acb1d40d4a34189e76ffa831c",
6093
            "placeholder": "​",
6094
            "style": "IPY_MODEL_1cbd0d65bcf9496c85e07225be76ffdf",
6095
            "value": "Downloading (…)olve/main/vocab.json: 100%"
6096
          }
6097
        },
6098
        "9439477ef5b64c1ba5c517d76c2c460f": {
6099
          "model_module": "@jupyter-widgets/controls",
6100
          "model_name": "FloatProgressModel",
6101
          "model_module_version": "1.5.0",
6102
          "state": {
6103
            "_dom_classes": [],
6104
            "_model_module": "@jupyter-widgets/controls",
6105
            "_model_module_version": "1.5.0",
6106
            "_model_name": "FloatProgressModel",
6107
            "_view_count": null,
6108
            "_view_module": "@jupyter-widgets/controls",
6109
            "_view_module_version": "1.5.0",
6110
            "_view_name": "ProgressView",
6111
            "bar_style": "success",
6112
            "description": "",
6113
            "description_tooltip": null,
6114
            "layout": "IPY_MODEL_4689e3a07503492eb11986bf5e8a864f",
6115
            "max": 798156,
6116
            "min": 0,
6117
            "orientation": "horizontal",
6118
            "style": "IPY_MODEL_4d443929f61f4991bc88caeb94adaaa3",
6119
            "value": 798156
6120
          }
6121
        },
6122
        "fc8787cef48c4befae588efa56f99adc": {
6123
          "model_module": "@jupyter-widgets/controls",
6124
          "model_name": "HTMLModel",
6125
          "model_module_version": "1.5.0",
6126
          "state": {
6127
            "_dom_classes": [],
6128
            "_model_module": "@jupyter-widgets/controls",
6129
            "_model_module_version": "1.5.0",
6130
            "_model_name": "HTMLModel",
6131
            "_view_count": null,
6132
            "_view_module": "@jupyter-widgets/controls",
6133
            "_view_module_version": "1.5.0",
6134
            "_view_name": "HTMLView",
6135
            "description": "",
6136
            "description_tooltip": null,
6137
            "layout": "IPY_MODEL_9f60e630e47841ba91855613da83e295",
6138
            "placeholder": "​",
6139
            "style": "IPY_MODEL_12332fa62a8f4b6f974eb37bf8eddaf2",
6140
            "value": " 798k/798k [00:00&lt;00:00, 16.8MB/s]"
6141
          }
6142
        },
6143
        "fd97aec74a1148b0b94de3f2194a0558": {
6144
          "model_module": "@jupyter-widgets/base",
6145
          "model_name": "LayoutModel",
6146
          "model_module_version": "1.2.0",
6147
          "state": {
6148
            "_model_module": "@jupyter-widgets/base",
6149
            "_model_module_version": "1.2.0",
6150
            "_model_name": "LayoutModel",
6151
            "_view_count": null,
6152
            "_view_module": "@jupyter-widgets/base",
6153
            "_view_module_version": "1.2.0",
6154
            "_view_name": "LayoutView",
6155
            "align_content": null,
6156
            "align_items": null,
6157
            "align_self": null,
6158
            "border": null,
6159
            "bottom": null,
6160
            "display": null,
6161
            "flex": null,
6162
            "flex_flow": null,
6163
            "grid_area": null,
6164
            "grid_auto_columns": null,
6165
            "grid_auto_flow": null,
6166
            "grid_auto_rows": null,
6167
            "grid_column": null,
6168
            "grid_gap": null,
6169
            "grid_row": null,
6170
            "grid_template_areas": null,
6171
            "grid_template_columns": null,
6172
            "grid_template_rows": null,
6173
            "height": null,
6174
            "justify_content": null,
6175
            "justify_items": null,
6176
            "left": null,
6177
            "margin": null,
6178
            "max_height": null,
6179
            "max_width": null,
6180
            "min_height": null,
6181
            "min_width": null,
6182
            "object_fit": null,
6183
            "object_position": null,
6184
            "order": null,
6185
            "overflow": null,
6186
            "overflow_x": null,
6187
            "overflow_y": null,
6188
            "padding": null,
6189
            "right": null,
6190
            "top": null,
6191
            "visibility": null,
6192
            "width": null
6193
          }
6194
        },
6195
        "cf54b45acb1d40d4a34189e76ffa831c": {
6196
          "model_module": "@jupyter-widgets/base",
6197
          "model_name": "LayoutModel",
6198
          "model_module_version": "1.2.0",
6199
          "state": {
6200
            "_model_module": "@jupyter-widgets/base",
6201
            "_model_module_version": "1.2.0",
6202
            "_model_name": "LayoutModel",
6203
            "_view_count": null,
6204
            "_view_module": "@jupyter-widgets/base",
6205
            "_view_module_version": "1.2.0",
6206
            "_view_name": "LayoutView",
6207
            "align_content": null,
6208
            "align_items": null,
6209
            "align_self": null,
6210
            "border": null,
6211
            "bottom": null,
6212
            "display": null,
6213
            "flex": null,
6214
            "flex_flow": null,
6215
            "grid_area": null,
6216
            "grid_auto_columns": null,
6217
            "grid_auto_flow": null,
6218
            "grid_auto_rows": null,
6219
            "grid_column": null,
6220
            "grid_gap": null,
6221
            "grid_row": null,
6222
            "grid_template_areas": null,
6223
            "grid_template_columns": null,
6224
            "grid_template_rows": null,
6225
            "height": null,
6226
            "justify_content": null,
6227
            "justify_items": null,
6228
            "left": null,
6229
            "margin": null,
6230
            "max_height": null,
6231
            "max_width": null,
6232
            "min_height": null,
6233
            "min_width": null,
6234
            "object_fit": null,
6235
            "object_position": null,
6236
            "order": null,
6237
            "overflow": null,
6238
            "overflow_x": null,
6239
            "overflow_y": null,
6240
            "padding": null,
6241
            "right": null,
6242
            "top": null,
6243
            "visibility": null,
6244
            "width": null
6245
          }
6246
        },
6247
        "1cbd0d65bcf9496c85e07225be76ffdf": {
6248
          "model_module": "@jupyter-widgets/controls",
6249
          "model_name": "DescriptionStyleModel",
6250
          "model_module_version": "1.5.0",
6251
          "state": {
6252
            "_model_module": "@jupyter-widgets/controls",
6253
            "_model_module_version": "1.5.0",
6254
            "_model_name": "DescriptionStyleModel",
6255
            "_view_count": null,
6256
            "_view_module": "@jupyter-widgets/base",
6257
            "_view_module_version": "1.2.0",
6258
            "_view_name": "StyleView",
6259
            "description_width": ""
6260
          }
6261
        },
6262
        "4689e3a07503492eb11986bf5e8a864f": {
6263
          "model_module": "@jupyter-widgets/base",
6264
          "model_name": "LayoutModel",
6265
          "model_module_version": "1.2.0",
6266
          "state": {
6267
            "_model_module": "@jupyter-widgets/base",
6268
            "_model_module_version": "1.2.0",
6269
            "_model_name": "LayoutModel",
6270
            "_view_count": null,
6271
            "_view_module": "@jupyter-widgets/base",
6272
            "_view_module_version": "1.2.0",
6273
            "_view_name": "LayoutView",
6274
            "align_content": null,
6275
            "align_items": null,
6276
            "align_self": null,
6277
            "border": null,
6278
            "bottom": null,
6279
            "display": null,
6280
            "flex": null,
6281
            "flex_flow": null,
6282
            "grid_area": null,
6283
            "grid_auto_columns": null,
6284
            "grid_auto_flow": null,
6285
            "grid_auto_rows": null,
6286
            "grid_column": null,
6287
            "grid_gap": null,
6288
            "grid_row": null,
6289
            "grid_template_areas": null,
6290
            "grid_template_columns": null,
6291
            "grid_template_rows": null,
6292
            "height": null,
6293
            "justify_content": null,
6294
            "justify_items": null,
6295
            "left": null,
6296
            "margin": null,
6297
            "max_height": null,
6298
            "max_width": null,
6299
            "min_height": null,
6300
            "min_width": null,
6301
            "object_fit": null,
6302
            "object_position": null,
6303
            "order": null,
6304
            "overflow": null,
6305
            "overflow_x": null,
6306
            "overflow_y": null,
6307
            "padding": null,
6308
            "right": null,
6309
            "top": null,
6310
            "visibility": null,
6311
            "width": null
6312
          }
6313
        },
6314
        "4d443929f61f4991bc88caeb94adaaa3": {
6315
          "model_module": "@jupyter-widgets/controls",
6316
          "model_name": "ProgressStyleModel",
6317
          "model_module_version": "1.5.0",
6318
          "state": {
6319
            "_model_module": "@jupyter-widgets/controls",
6320
            "_model_module_version": "1.5.0",
6321
            "_model_name": "ProgressStyleModel",
6322
            "_view_count": null,
6323
            "_view_module": "@jupyter-widgets/base",
6324
            "_view_module_version": "1.2.0",
6325
            "_view_name": "StyleView",
6326
            "bar_color": null,
6327
            "description_width": ""
6328
          }
6329
        },
6330
        "9f60e630e47841ba91855613da83e295": {
6331
          "model_module": "@jupyter-widgets/base",
6332
          "model_name": "LayoutModel",
6333
          "model_module_version": "1.2.0",
6334
          "state": {
6335
            "_model_module": "@jupyter-widgets/base",
6336
            "_model_module_version": "1.2.0",
6337
            "_model_name": "LayoutModel",
6338
            "_view_count": null,
6339
            "_view_module": "@jupyter-widgets/base",
6340
            "_view_module_version": "1.2.0",
6341
            "_view_name": "LayoutView",
6342
            "align_content": null,
6343
            "align_items": null,
6344
            "align_self": null,
6345
            "border": null,
6346
            "bottom": null,
6347
            "display": null,
6348
            "flex": null,
6349
            "flex_flow": null,
6350
            "grid_area": null,
6351
            "grid_auto_columns": null,
6352
            "grid_auto_flow": null,
6353
            "grid_auto_rows": null,
6354
            "grid_column": null,
6355
            "grid_gap": null,
6356
            "grid_row": null,
6357
            "grid_template_areas": null,
6358
            "grid_template_columns": null,
6359
            "grid_template_rows": null,
6360
            "height": null,
6361
            "justify_content": null,
6362
            "justify_items": null,
6363
            "left": null,
6364
            "margin": null,
6365
            "max_height": null,
6366
            "max_width": null,
6367
            "min_height": null,
6368
            "min_width": null,
6369
            "object_fit": null,
6370
            "object_position": null,
6371
            "order": null,
6372
            "overflow": null,
6373
            "overflow_x": null,
6374
            "overflow_y": null,
6375
            "padding": null,
6376
            "right": null,
6377
            "top": null,
6378
            "visibility": null,
6379
            "width": null
6380
          }
6381
        },
6382
        "12332fa62a8f4b6f974eb37bf8eddaf2": {
6383
          "model_module": "@jupyter-widgets/controls",
6384
          "model_name": "DescriptionStyleModel",
6385
          "model_module_version": "1.5.0",
6386
          "state": {
6387
            "_model_module": "@jupyter-widgets/controls",
6388
            "_model_module_version": "1.5.0",
6389
            "_model_name": "DescriptionStyleModel",
6390
            "_view_count": null,
6391
            "_view_module": "@jupyter-widgets/base",
6392
            "_view_module_version": "1.2.0",
6393
            "_view_name": "StyleView",
6394
            "description_width": ""
6395
          }
6396
        },
6397
        "e333b53d363b47648ab6694e25b68ad7": {
6398
          "model_module": "@jupyter-widgets/controls",
6399
          "model_name": "HBoxModel",
6400
          "model_module_version": "1.5.0",
6401
          "state": {
6402
            "_dom_classes": [],
6403
            "_model_module": "@jupyter-widgets/controls",
6404
            "_model_module_version": "1.5.0",
6405
            "_model_name": "HBoxModel",
6406
            "_view_count": null,
6407
            "_view_module": "@jupyter-widgets/controls",
6408
            "_view_module_version": "1.5.0",
6409
            "_view_name": "HBoxView",
6410
            "box_style": "",
6411
            "children": [
6412
              "IPY_MODEL_5e9ebdfb88fa413b88fdf2dd5bc2a3a1",
6413
              "IPY_MODEL_058ff0fedb9545199d11930a9aa1deec",
6414
              "IPY_MODEL_6996b093bc5a4ba385dfd134f687e8db"
6415
            ],
6416
            "layout": "IPY_MODEL_e1a107dbb4784261b608c382895e7daf"
6417
          }
6418
        },
6419
        "5e9ebdfb88fa413b88fdf2dd5bc2a3a1": {
6420
          "model_module": "@jupyter-widgets/controls",
6421
          "model_name": "HTMLModel",
6422
          "model_module_version": "1.5.0",
6423
          "state": {
6424
            "_dom_classes": [],
6425
            "_model_module": "@jupyter-widgets/controls",
6426
            "_model_module_version": "1.5.0",
6427
            "_model_name": "HTMLModel",
6428
            "_view_count": null,
6429
            "_view_module": "@jupyter-widgets/controls",
6430
            "_view_module_version": "1.5.0",
6431
            "_view_name": "HTMLView",
6432
            "description": "",
6433
            "description_tooltip": null,
6434
            "layout": "IPY_MODEL_73ef312d625542ccaf7994e9300acb1b",
6435
            "placeholder": "​",
6436
            "style": "IPY_MODEL_541a328e5cb84163a1295eeee4494d3f",
6437
            "value": "Downloading (…)olve/main/merges.txt: 100%"
6438
          }
6439
        },
6440
        "058ff0fedb9545199d11930a9aa1deec": {
6441
          "model_module": "@jupyter-widgets/controls",
6442
          "model_name": "FloatProgressModel",
6443
          "model_module_version": "1.5.0",
6444
          "state": {
6445
            "_dom_classes": [],
6446
            "_model_module": "@jupyter-widgets/controls",
6447
            "_model_module_version": "1.5.0",
6448
            "_model_name": "FloatProgressModel",
6449
            "_view_count": null,
6450
            "_view_module": "@jupyter-widgets/controls",
6451
            "_view_module_version": "1.5.0",
6452
            "_view_name": "ProgressView",
6453
            "bar_style": "success",
6454
            "description": "",
6455
            "description_tooltip": null,
6456
            "layout": "IPY_MODEL_db228a2b64c148d98f5db9bfbf2ac5af",
6457
            "max": 456318,
6458
            "min": 0,
6459
            "orientation": "horizontal",
6460
            "style": "IPY_MODEL_85455342596f4e90b98cc41da60b8cb6",
6461
            "value": 456318
6462
          }
6463
        },
6464
        "6996b093bc5a4ba385dfd134f687e8db": {
6465
          "model_module": "@jupyter-widgets/controls",
6466
          "model_name": "HTMLModel",
6467
          "model_module_version": "1.5.0",
6468
          "state": {
6469
            "_dom_classes": [],
6470
            "_model_module": "@jupyter-widgets/controls",
6471
            "_model_module_version": "1.5.0",
6472
            "_model_name": "HTMLModel",
6473
            "_view_count": null,
6474
            "_view_module": "@jupyter-widgets/controls",
6475
            "_view_module_version": "1.5.0",
6476
            "_view_name": "HTMLView",
6477
            "description": "",
6478
            "description_tooltip": null,
6479
            "layout": "IPY_MODEL_22b767cd81db4953959e78d71f604dce",
6480
            "placeholder": "​",
6481
            "style": "IPY_MODEL_9fb8334f33e84dc3a4bb3deaf1e7777c",
6482
            "value": " 456k/456k [00:00&lt;00:00, 8.57MB/s]"
6483
          }
6484
        },
6485
        "e1a107dbb4784261b608c382895e7daf": {
6486
          "model_module": "@jupyter-widgets/base",
6487
          "model_name": "LayoutModel",
6488
          "model_module_version": "1.2.0",
6489
          "state": {
6490
            "_model_module": "@jupyter-widgets/base",
6491
            "_model_module_version": "1.2.0",
6492
            "_model_name": "LayoutModel",
6493
            "_view_count": null,
6494
            "_view_module": "@jupyter-widgets/base",
6495
            "_view_module_version": "1.2.0",
6496
            "_view_name": "LayoutView",
6497
            "align_content": null,
6498
            "align_items": null,
6499
            "align_self": null,
6500
            "border": null,
6501
            "bottom": null,
6502
            "display": null,
6503
            "flex": null,
6504
            "flex_flow": null,
6505
            "grid_area": null,
6506
            "grid_auto_columns": null,
6507
            "grid_auto_flow": null,
6508
            "grid_auto_rows": null,
6509
            "grid_column": null,
6510
            "grid_gap": null,
6511
            "grid_row": null,
6512
            "grid_template_areas": null,
6513
            "grid_template_columns": null,
6514
            "grid_template_rows": null,
6515
            "height": null,
6516
            "justify_content": null,
6517
            "justify_items": null,
6518
            "left": null,
6519
            "margin": null,
6520
            "max_height": null,
6521
            "max_width": null,
6522
            "min_height": null,
6523
            "min_width": null,
6524
            "object_fit": null,
6525
            "object_position": null,
6526
            "order": null,
6527
            "overflow": null,
6528
            "overflow_x": null,
6529
            "overflow_y": null,
6530
            "padding": null,
6531
            "right": null,
6532
            "top": null,
6533
            "visibility": null,
6534
            "width": null
6535
          }
6536
        },
6537
        "73ef312d625542ccaf7994e9300acb1b": {
6538
          "model_module": "@jupyter-widgets/base",
6539
          "model_name": "LayoutModel",
6540
          "model_module_version": "1.2.0",
6541
          "state": {
6542
            "_model_module": "@jupyter-widgets/base",
6543
            "_model_module_version": "1.2.0",
6544
            "_model_name": "LayoutModel",
6545
            "_view_count": null,
6546
            "_view_module": "@jupyter-widgets/base",
6547
            "_view_module_version": "1.2.0",
6548
            "_view_name": "LayoutView",
6549
            "align_content": null,
6550
            "align_items": null,
6551
            "align_self": null,
6552
            "border": null,
6553
            "bottom": null,
6554
            "display": null,
6555
            "flex": null,
6556
            "flex_flow": null,
6557
            "grid_area": null,
6558
            "grid_auto_columns": null,
6559
            "grid_auto_flow": null,
6560
            "grid_auto_rows": null,
6561
            "grid_column": null,
6562
            "grid_gap": null,
6563
            "grid_row": null,
6564
            "grid_template_areas": null,
6565
            "grid_template_columns": null,
6566
            "grid_template_rows": null,
6567
            "height": null,
6568
            "justify_content": null,
6569
            "justify_items": null,
6570
            "left": null,
6571
            "margin": null,
6572
            "max_height": null,
6573
            "max_width": null,
6574
            "min_height": null,
6575
            "min_width": null,
6576
            "object_fit": null,
6577
            "object_position": null,
6578
            "order": null,
6579
            "overflow": null,
6580
            "overflow_x": null,
6581
            "overflow_y": null,
6582
            "padding": null,
6583
            "right": null,
6584
            "top": null,
6585
            "visibility": null,
6586
            "width": null
6587
          }
6588
        },
6589
        "541a328e5cb84163a1295eeee4494d3f": {
6590
          "model_module": "@jupyter-widgets/controls",
6591
          "model_name": "DescriptionStyleModel",
6592
          "model_module_version": "1.5.0",
6593
          "state": {
6594
            "_model_module": "@jupyter-widgets/controls",
6595
            "_model_module_version": "1.5.0",
6596
            "_model_name": "DescriptionStyleModel",
6597
            "_view_count": null,
6598
            "_view_module": "@jupyter-widgets/base",
6599
            "_view_module_version": "1.2.0",
6600
            "_view_name": "StyleView",
6601
            "description_width": ""
6602
          }
6603
        },
6604
        "db228a2b64c148d98f5db9bfbf2ac5af": {
6605
          "model_module": "@jupyter-widgets/base",
6606
          "model_name": "LayoutModel",
6607
          "model_module_version": "1.2.0",
6608
          "state": {
6609
            "_model_module": "@jupyter-widgets/base",
6610
            "_model_module_version": "1.2.0",
6611
            "_model_name": "LayoutModel",
6612
            "_view_count": null,
6613
            "_view_module": "@jupyter-widgets/base",
6614
            "_view_module_version": "1.2.0",
6615
            "_view_name": "LayoutView",
6616
            "align_content": null,
6617
            "align_items": null,
6618
            "align_self": null,
6619
            "border": null,
6620
            "bottom": null,
6621
            "display": null,
6622
            "flex": null,
6623
            "flex_flow": null,
6624
            "grid_area": null,
6625
            "grid_auto_columns": null,
6626
            "grid_auto_flow": null,
6627
            "grid_auto_rows": null,
6628
            "grid_column": null,
6629
            "grid_gap": null,
6630
            "grid_row": null,
6631
            "grid_template_areas": null,
6632
            "grid_template_columns": null,
6633
            "grid_template_rows": null,
6634
            "height": null,
6635
            "justify_content": null,
6636
            "justify_items": null,
6637
            "left": null,
6638
            "margin": null,
6639
            "max_height": null,
6640
            "max_width": null,
6641
            "min_height": null,
6642
            "min_width": null,
6643
            "object_fit": null,
6644
            "object_position": null,
6645
            "order": null,
6646
            "overflow": null,
6647
            "overflow_x": null,
6648
            "overflow_y": null,
6649
            "padding": null,
6650
            "right": null,
6651
            "top": null,
6652
            "visibility": null,
6653
            "width": null
6654
          }
6655
        },
6656
        "85455342596f4e90b98cc41da60b8cb6": {
6657
          "model_module": "@jupyter-widgets/controls",
6658
          "model_name": "ProgressStyleModel",
6659
          "model_module_version": "1.5.0",
6660
          "state": {
6661
            "_model_module": "@jupyter-widgets/controls",
6662
            "_model_module_version": "1.5.0",
6663
            "_model_name": "ProgressStyleModel",
6664
            "_view_count": null,
6665
            "_view_module": "@jupyter-widgets/base",
6666
            "_view_module_version": "1.2.0",
6667
            "_view_name": "StyleView",
6668
            "bar_color": null,
6669
            "description_width": ""
6670
          }
6671
        },
6672
        "22b767cd81db4953959e78d71f604dce": {
6673
          "model_module": "@jupyter-widgets/base",
6674
          "model_name": "LayoutModel",
6675
          "model_module_version": "1.2.0",
6676
          "state": {
6677
            "_model_module": "@jupyter-widgets/base",
6678
            "_model_module_version": "1.2.0",
6679
            "_model_name": "LayoutModel",
6680
            "_view_count": null,
6681
            "_view_module": "@jupyter-widgets/base",
6682
            "_view_module_version": "1.2.0",
6683
            "_view_name": "LayoutView",
6684
            "align_content": null,
6685
            "align_items": null,
6686
            "align_self": null,
6687
            "border": null,
6688
            "bottom": null,
6689
            "display": null,
6690
            "flex": null,
6691
            "flex_flow": null,
6692
            "grid_area": null,
6693
            "grid_auto_columns": null,
6694
            "grid_auto_flow": null,
6695
            "grid_auto_rows": null,
6696
            "grid_column": null,
6697
            "grid_gap": null,
6698
            "grid_row": null,
6699
            "grid_template_areas": null,
6700
            "grid_template_columns": null,
6701
            "grid_template_rows": null,
6702
            "height": null,
6703
            "justify_content": null,
6704
            "justify_items": null,
6705
            "left": null,
6706
            "margin": null,
6707
            "max_height": null,
6708
            "max_width": null,
6709
            "min_height": null,
6710
            "min_width": null,
6711
            "object_fit": null,
6712
            "object_position": null,
6713
            "order": null,
6714
            "overflow": null,
6715
            "overflow_x": null,
6716
            "overflow_y": null,
6717
            "padding": null,
6718
            "right": null,
6719
            "top": null,
6720
            "visibility": null,
6721
            "width": null
6722
          }
6723
        },
6724
        "9fb8334f33e84dc3a4bb3deaf1e7777c": {
6725
          "model_module": "@jupyter-widgets/controls",
6726
          "model_name": "DescriptionStyleModel",
6727
          "model_module_version": "1.5.0",
6728
          "state": {
6729
            "_model_module": "@jupyter-widgets/controls",
6730
            "_model_module_version": "1.5.0",
6731
            "_model_name": "DescriptionStyleModel",
6732
            "_view_count": null,
6733
            "_view_module": "@jupyter-widgets/base",
6734
            "_view_module_version": "1.2.0",
6735
            "_view_name": "StyleView",
6736
            "description_width": ""
6737
          }
6738
        },
6739
        "f6ee0db62fe14f1e86c972a4a157e315": {
6740
          "model_module": "@jupyter-widgets/controls",
6741
          "model_name": "HBoxModel",
6742
          "model_module_version": "1.5.0",
6743
          "state": {
6744
            "_dom_classes": [],
6745
            "_model_module": "@jupyter-widgets/controls",
6746
            "_model_module_version": "1.5.0",
6747
            "_model_name": "HBoxModel",
6748
            "_view_count": null,
6749
            "_view_module": "@jupyter-widgets/controls",
6750
            "_view_module_version": "1.5.0",
6751
            "_view_name": "HBoxView",
6752
            "box_style": "",
6753
            "children": [
6754
              "IPY_MODEL_f57698a07edd4a43b0b1f55e0d6b4a02",
6755
              "IPY_MODEL_9adaf92c8acf4b7ea94e5591cfe27707",
6756
              "IPY_MODEL_f1d7c8a0a78041b184d64a69f7ed0789"
6757
            ],
6758
            "layout": "IPY_MODEL_f4ca5f16039b4258b9c83768d211e757"
6759
          }
6760
        },
6761
        "f57698a07edd4a43b0b1f55e0d6b4a02": {
6762
          "model_module": "@jupyter-widgets/controls",
6763
          "model_name": "HTMLModel",
6764
          "model_module_version": "1.5.0",
6765
          "state": {
6766
            "_dom_classes": [],
6767
            "_model_module": "@jupyter-widgets/controls",
6768
            "_model_module_version": "1.5.0",
6769
            "_model_name": "HTMLModel",
6770
            "_view_count": null,
6771
            "_view_module": "@jupyter-widgets/controls",
6772
            "_view_module_version": "1.5.0",
6773
            "_view_name": "HTMLView",
6774
            "description": "",
6775
            "description_tooltip": null,
6776
            "layout": "IPY_MODEL_864288a708f545629900a29fb05f3d8b",
6777
            "placeholder": "​",
6778
            "style": "IPY_MODEL_746c0bfc3d90446ca10a16d7838441aa",
6779
            "value": "Downloading (…)/main/tokenizer.json: 100%"
6780
          }
6781
        },
6782
        "9adaf92c8acf4b7ea94e5591cfe27707": {
6783
          "model_module": "@jupyter-widgets/controls",
6784
          "model_name": "FloatProgressModel",
6785
          "model_module_version": "1.5.0",
6786
          "state": {
6787
            "_dom_classes": [],
6788
            "_model_module": "@jupyter-widgets/controls",
6789
            "_model_module_version": "1.5.0",
6790
            "_model_name": "FloatProgressModel",
6791
            "_view_count": null,
6792
            "_view_module": "@jupyter-widgets/controls",
6793
            "_view_module_version": "1.5.0",
6794
            "_view_name": "ProgressView",
6795
            "bar_style": "success",
6796
            "description": "",
6797
            "description_tooltip": null,
6798
            "layout": "IPY_MODEL_889a67c857fe40c89d0d8e523a66ae08",
6799
            "max": 2114924,
6800
            "min": 0,
6801
            "orientation": "horizontal",
6802
            "style": "IPY_MODEL_f66ce8552d5d4355b90e934a1c8c04cd",
6803
            "value": 2114924
6804
          }
6805
        },
6806
        "f1d7c8a0a78041b184d64a69f7ed0789": {
6807
          "model_module": "@jupyter-widgets/controls",
6808
          "model_name": "HTMLModel",
6809
          "model_module_version": "1.5.0",
6810
          "state": {
6811
            "_dom_classes": [],
6812
            "_model_module": "@jupyter-widgets/controls",
6813
            "_model_module_version": "1.5.0",
6814
            "_model_name": "HTMLModel",
6815
            "_view_count": null,
6816
            "_view_module": "@jupyter-widgets/controls",
6817
            "_view_module_version": "1.5.0",
6818
            "_view_name": "HTMLView",
6819
            "description": "",
6820
            "description_tooltip": null,
6821
            "layout": "IPY_MODEL_0c3a38f76e6841a38f7a0e545c9c4211",
6822
            "placeholder": "​",
6823
            "style": "IPY_MODEL_e75c5a29c59747dda108212a336c7e2c",
6824
            "value": " 2.11M/2.11M [00:00&lt;00:00, 26.6MB/s]"
6825
          }
6826
        },
6827
        "f4ca5f16039b4258b9c83768d211e757": {
6828
          "model_module": "@jupyter-widgets/base",
6829
          "model_name": "LayoutModel",
6830
          "model_module_version": "1.2.0",
6831
          "state": {
6832
            "_model_module": "@jupyter-widgets/base",
6833
            "_model_module_version": "1.2.0",
6834
            "_model_name": "LayoutModel",
6835
            "_view_count": null,
6836
            "_view_module": "@jupyter-widgets/base",
6837
            "_view_module_version": "1.2.0",
6838
            "_view_name": "LayoutView",
6839
            "align_content": null,
6840
            "align_items": null,
6841
            "align_self": null,
6842
            "border": null,
6843
            "bottom": null,
6844
            "display": null,
6845
            "flex": null,
6846
            "flex_flow": null,
6847
            "grid_area": null,
6848
            "grid_auto_columns": null,
6849
            "grid_auto_flow": null,
6850
            "grid_auto_rows": null,
6851
            "grid_column": null,
6852
            "grid_gap": null,
6853
            "grid_row": null,
6854
            "grid_template_areas": null,
6855
            "grid_template_columns": null,
6856
            "grid_template_rows": null,
6857
            "height": null,
6858
            "justify_content": null,
6859
            "justify_items": null,
6860
            "left": null,
6861
            "margin": null,
6862
            "max_height": null,
6863
            "max_width": null,
6864
            "min_height": null,
6865
            "min_width": null,
6866
            "object_fit": null,
6867
            "object_position": null,
6868
            "order": null,
6869
            "overflow": null,
6870
            "overflow_x": null,
6871
            "overflow_y": null,
6872
            "padding": null,
6873
            "right": null,
6874
            "top": null,
6875
            "visibility": null,
6876
            "width": null
6877
          }
6878
        },
6879
        "864288a708f545629900a29fb05f3d8b": {
6880
          "model_module": "@jupyter-widgets/base",
6881
          "model_name": "LayoutModel",
6882
          "model_module_version": "1.2.0",
6883
          "state": {
6884
            "_model_module": "@jupyter-widgets/base",
6885
            "_model_module_version": "1.2.0",
6886
            "_model_name": "LayoutModel",
6887
            "_view_count": null,
6888
            "_view_module": "@jupyter-widgets/base",
6889
            "_view_module_version": "1.2.0",
6890
            "_view_name": "LayoutView",
6891
            "align_content": null,
6892
            "align_items": null,
6893
            "align_self": null,
6894
            "border": null,
6895
            "bottom": null,
6896
            "display": null,
6897
            "flex": null,
6898
            "flex_flow": null,
6899
            "grid_area": null,
6900
            "grid_auto_columns": null,
6901
            "grid_auto_flow": null,
6902
            "grid_auto_rows": null,
6903
            "grid_column": null,
6904
            "grid_gap": null,
6905
            "grid_row": null,
6906
            "grid_template_areas": null,
6907
            "grid_template_columns": null,
6908
            "grid_template_rows": null,
6909
            "height": null,
6910
            "justify_content": null,
6911
            "justify_items": null,
6912
            "left": null,
6913
            "margin": null,
6914
            "max_height": null,
6915
            "max_width": null,
6916
            "min_height": null,
6917
            "min_width": null,
6918
            "object_fit": null,
6919
            "object_position": null,
6920
            "order": null,
6921
            "overflow": null,
6922
            "overflow_x": null,
6923
            "overflow_y": null,
6924
            "padding": null,
6925
            "right": null,
6926
            "top": null,
6927
            "visibility": null,
6928
            "width": null
6929
          }
6930
        },
6931
        "746c0bfc3d90446ca10a16d7838441aa": {
6932
          "model_module": "@jupyter-widgets/controls",
6933
          "model_name": "DescriptionStyleModel",
6934
          "model_module_version": "1.5.0",
6935
          "state": {
6936
            "_model_module": "@jupyter-widgets/controls",
6937
            "_model_module_version": "1.5.0",
6938
            "_model_name": "DescriptionStyleModel",
6939
            "_view_count": null,
6940
            "_view_module": "@jupyter-widgets/base",
6941
            "_view_module_version": "1.2.0",
6942
            "_view_name": "StyleView",
6943
            "description_width": ""
6944
          }
6945
        },
6946
        "889a67c857fe40c89d0d8e523a66ae08": {
6947
          "model_module": "@jupyter-widgets/base",
6948
          "model_name": "LayoutModel",
6949
          "model_module_version": "1.2.0",
6950
          "state": {
6951
            "_model_module": "@jupyter-widgets/base",
6952
            "_model_module_version": "1.2.0",
6953
            "_model_name": "LayoutModel",
6954
            "_view_count": null,
6955
            "_view_module": "@jupyter-widgets/base",
6956
            "_view_module_version": "1.2.0",
6957
            "_view_name": "LayoutView",
6958
            "align_content": null,
6959
            "align_items": null,
6960
            "align_self": null,
6961
            "border": null,
6962
            "bottom": null,
6963
            "display": null,
6964
            "flex": null,
6965
            "flex_flow": null,
6966
            "grid_area": null,
6967
            "grid_auto_columns": null,
6968
            "grid_auto_flow": null,
6969
            "grid_auto_rows": null,
6970
            "grid_column": null,
6971
            "grid_gap": null,
6972
            "grid_row": null,
6973
            "grid_template_areas": null,
6974
            "grid_template_columns": null,
6975
            "grid_template_rows": null,
6976
            "height": null,
6977
            "justify_content": null,
6978
            "justify_items": null,
6979
            "left": null,
6980
            "margin": null,
6981
            "max_height": null,
6982
            "max_width": null,
6983
            "min_height": null,
6984
            "min_width": null,
6985
            "object_fit": null,
6986
            "object_position": null,
6987
            "order": null,
6988
            "overflow": null,
6989
            "overflow_x": null,
6990
            "overflow_y": null,
6991
            "padding": null,
6992
            "right": null,
6993
            "top": null,
6994
            "visibility": null,
6995
            "width": null
6996
          }
6997
        },
6998
        "f66ce8552d5d4355b90e934a1c8c04cd": {
6999
          "model_module": "@jupyter-widgets/controls",
7000
          "model_name": "ProgressStyleModel",
7001
          "model_module_version": "1.5.0",
7002
          "state": {
7003
            "_model_module": "@jupyter-widgets/controls",
7004
            "_model_module_version": "1.5.0",
7005
            "_model_name": "ProgressStyleModel",
7006
            "_view_count": null,
7007
            "_view_module": "@jupyter-widgets/base",
7008
            "_view_module_version": "1.2.0",
7009
            "_view_name": "StyleView",
7010
            "bar_color": null,
7011
            "description_width": ""
7012
          }
7013
        },
7014
        "0c3a38f76e6841a38f7a0e545c9c4211": {
7015
          "model_module": "@jupyter-widgets/base",
7016
          "model_name": "LayoutModel",
7017
          "model_module_version": "1.2.0",
7018
          "state": {
7019
            "_model_module": "@jupyter-widgets/base",
7020
            "_model_module_version": "1.2.0",
7021
            "_model_name": "LayoutModel",
7022
            "_view_count": null,
7023
            "_view_module": "@jupyter-widgets/base",
7024
            "_view_module_version": "1.2.0",
7025
            "_view_name": "LayoutView",
7026
            "align_content": null,
7027
            "align_items": null,
7028
            "align_self": null,
7029
            "border": null,
7030
            "bottom": null,
7031
            "display": null,
7032
            "flex": null,
7033
            "flex_flow": null,
7034
            "grid_area": null,
7035
            "grid_auto_columns": null,
7036
            "grid_auto_flow": null,
7037
            "grid_auto_rows": null,
7038
            "grid_column": null,
7039
            "grid_gap": null,
7040
            "grid_row": null,
7041
            "grid_template_areas": null,
7042
            "grid_template_columns": null,
7043
            "grid_template_rows": null,
7044
            "height": null,
7045
            "justify_content": null,
7046
            "justify_items": null,
7047
            "left": null,
7048
            "margin": null,
7049
            "max_height": null,
7050
            "max_width": null,
7051
            "min_height": null,
7052
            "min_width": null,
7053
            "object_fit": null,
7054
            "object_position": null,
7055
            "order": null,
7056
            "overflow": null,
7057
            "overflow_x": null,
7058
            "overflow_y": null,
7059
            "padding": null,
7060
            "right": null,
7061
            "top": null,
7062
            "visibility": null,
7063
            "width": null
7064
          }
7065
        },
7066
        "e75c5a29c59747dda108212a336c7e2c": {
7067
          "model_module": "@jupyter-widgets/controls",
7068
          "model_name": "DescriptionStyleModel",
7069
          "model_module_version": "1.5.0",
7070
          "state": {
7071
            "_model_module": "@jupyter-widgets/controls",
7072
            "_model_module_version": "1.5.0",
7073
            "_model_name": "DescriptionStyleModel",
7074
            "_view_count": null,
7075
            "_view_module": "@jupyter-widgets/base",
7076
            "_view_module_version": "1.2.0",
7077
            "_view_name": "StyleView",
7078
            "description_width": ""
7079
          }
7080
        },
7081
        "ce6b953129c945c7a7066d0f07385c92": {
7082
          "model_module": "@jupyter-widgets/controls",
7083
          "model_name": "HBoxModel",
7084
          "model_module_version": "1.5.0",
7085
          "state": {
7086
            "_dom_classes": [],
7087
            "_model_module": "@jupyter-widgets/controls",
7088
            "_model_module_version": "1.5.0",
7089
            "_model_name": "HBoxModel",
7090
            "_view_count": null,
7091
            "_view_module": "@jupyter-widgets/controls",
7092
            "_view_module_version": "1.5.0",
7093
            "_view_name": "HBoxView",
7094
            "box_style": "",
7095
            "children": [
7096
              "IPY_MODEL_6c5a70913c6542be93085a3eefb2b1ce",
7097
              "IPY_MODEL_bd5a93537e9d49ec86233397c0ec2a6f",
7098
              "IPY_MODEL_36fad39ecf0347079291103b73c3f6dd"
7099
            ],
7100
            "layout": "IPY_MODEL_14f4fc13863648599cef5b830ab1bd9f"
7101
          }
7102
        },
7103
        "6c5a70913c6542be93085a3eefb2b1ce": {
7104
          "model_module": "@jupyter-widgets/controls",
7105
          "model_name": "HTMLModel",
7106
          "model_module_version": "1.5.0",
7107
          "state": {
7108
            "_dom_classes": [],
7109
            "_model_module": "@jupyter-widgets/controls",
7110
            "_model_module_version": "1.5.0",
7111
            "_model_name": "HTMLModel",
7112
            "_view_count": null,
7113
            "_view_module": "@jupyter-widgets/controls",
7114
            "_view_module_version": "1.5.0",
7115
            "_view_name": "HTMLView",
7116
            "description": "",
7117
            "description_tooltip": null,
7118
            "layout": "IPY_MODEL_4212106d48a34279a45890b62879ec87",
7119
            "placeholder": "​",
7120
            "style": "IPY_MODEL_2bee159b595643619607aa10b45c186f",
7121
            "value": "Downloading (…)in/added_tokens.json: 100%"
7122
          }
7123
        },
7124
        "bd5a93537e9d49ec86233397c0ec2a6f": {
7125
          "model_module": "@jupyter-widgets/controls",
7126
          "model_name": "FloatProgressModel",
7127
          "model_module_version": "1.5.0",
7128
          "state": {
7129
            "_dom_classes": [],
7130
            "_model_module": "@jupyter-widgets/controls",
7131
            "_model_module_version": "1.5.0",
7132
            "_model_name": "FloatProgressModel",
7133
            "_view_count": null,
7134
            "_view_module": "@jupyter-widgets/controls",
7135
            "_view_module_version": "1.5.0",
7136
            "_view_name": "ProgressView",
7137
            "bar_style": "success",
7138
            "description": "",
7139
            "description_tooltip": null,
7140
            "layout": "IPY_MODEL_c2502539d4eb413da194fae615d6e6bf",
7141
            "max": 1080,
7142
            "min": 0,
7143
            "orientation": "horizontal",
7144
            "style": "IPY_MODEL_95ab78e1349d466d90df3390c0a6ceff",
7145
            "value": 1080
7146
          }
7147
        },
7148
        "36fad39ecf0347079291103b73c3f6dd": {
7149
          "model_module": "@jupyter-widgets/controls",
7150
          "model_name": "HTMLModel",
7151
          "model_module_version": "1.5.0",
7152
          "state": {
7153
            "_dom_classes": [],
7154
            "_model_module": "@jupyter-widgets/controls",
7155
            "_model_module_version": "1.5.0",
7156
            "_model_name": "HTMLModel",
7157
            "_view_count": null,
7158
            "_view_module": "@jupyter-widgets/controls",
7159
            "_view_module_version": "1.5.0",
7160
            "_view_name": "HTMLView",
7161
            "description": "",
7162
            "description_tooltip": null,
7163
            "layout": "IPY_MODEL_9ca4b24b9dbb4b5a9af5897c46069c49",
7164
            "placeholder": "​",
7165
            "style": "IPY_MODEL_5e5449f0eda34f29903b59ca07ac189f",
7166
            "value": " 1.08k/1.08k [00:00&lt;00:00, 72.4kB/s]"
7167
          }
7168
        },
7169
        "14f4fc13863648599cef5b830ab1bd9f": {
7170
          "model_module": "@jupyter-widgets/base",
7171
          "model_name": "LayoutModel",
7172
          "model_module_version": "1.2.0",
7173
          "state": {
7174
            "_model_module": "@jupyter-widgets/base",
7175
            "_model_module_version": "1.2.0",
7176
            "_model_name": "LayoutModel",
7177
            "_view_count": null,
7178
            "_view_module": "@jupyter-widgets/base",
7179
            "_view_module_version": "1.2.0",
7180
            "_view_name": "LayoutView",
7181
            "align_content": null,
7182
            "align_items": null,
7183
            "align_self": null,
7184
            "border": null,
7185
            "bottom": null,
7186
            "display": null,
7187
            "flex": null,
7188
            "flex_flow": null,
7189
            "grid_area": null,
7190
            "grid_auto_columns": null,
7191
            "grid_auto_flow": null,
7192
            "grid_auto_rows": null,
7193
            "grid_column": null,
7194
            "grid_gap": null,
7195
            "grid_row": null,
7196
            "grid_template_areas": null,
7197
            "grid_template_columns": null,
7198
            "grid_template_rows": null,
7199
            "height": null,
7200
            "justify_content": null,
7201
            "justify_items": null,
7202
            "left": null,
7203
            "margin": null,
7204
            "max_height": null,
7205
            "max_width": null,
7206
            "min_height": null,
7207
            "min_width": null,
7208
            "object_fit": null,
7209
            "object_position": null,
7210
            "order": null,
7211
            "overflow": null,
7212
            "overflow_x": null,
7213
            "overflow_y": null,
7214
            "padding": null,
7215
            "right": null,
7216
            "top": null,
7217
            "visibility": null,
7218
            "width": null
7219
          }
7220
        },
7221
        "4212106d48a34279a45890b62879ec87": {
7222
          "model_module": "@jupyter-widgets/base",
7223
          "model_name": "LayoutModel",
7224
          "model_module_version": "1.2.0",
7225
          "state": {
7226
            "_model_module": "@jupyter-widgets/base",
7227
            "_model_module_version": "1.2.0",
7228
            "_model_name": "LayoutModel",
7229
            "_view_count": null,
7230
            "_view_module": "@jupyter-widgets/base",
7231
            "_view_module_version": "1.2.0",
7232
            "_view_name": "LayoutView",
7233
            "align_content": null,
7234
            "align_items": null,
7235
            "align_self": null,
7236
            "border": null,
7237
            "bottom": null,
7238
            "display": null,
7239
            "flex": null,
7240
            "flex_flow": null,
7241
            "grid_area": null,
7242
            "grid_auto_columns": null,
7243
            "grid_auto_flow": null,
7244
            "grid_auto_rows": null,
7245
            "grid_column": null,
7246
            "grid_gap": null,
7247
            "grid_row": null,
7248
            "grid_template_areas": null,
7249
            "grid_template_columns": null,
7250
            "grid_template_rows": null,
7251
            "height": null,
7252
            "justify_content": null,
7253
            "justify_items": null,
7254
            "left": null,
7255
            "margin": null,
7256
            "max_height": null,
7257
            "max_width": null,
7258
            "min_height": null,
7259
            "min_width": null,
7260
            "object_fit": null,
7261
            "object_position": null,
7262
            "order": null,
7263
            "overflow": null,
7264
            "overflow_x": null,
7265
            "overflow_y": null,
7266
            "padding": null,
7267
            "right": null,
7268
            "top": null,
7269
            "visibility": null,
7270
            "width": null
7271
          }
7272
        },
7273
        "2bee159b595643619607aa10b45c186f": {
7274
          "model_module": "@jupyter-widgets/controls",
7275
          "model_name": "DescriptionStyleModel",
7276
          "model_module_version": "1.5.0",
7277
          "state": {
7278
            "_model_module": "@jupyter-widgets/controls",
7279
            "_model_module_version": "1.5.0",
7280
            "_model_name": "DescriptionStyleModel",
7281
            "_view_count": null,
7282
            "_view_module": "@jupyter-widgets/base",
7283
            "_view_module_version": "1.2.0",
7284
            "_view_name": "StyleView",
7285
            "description_width": ""
7286
          }
7287
        },
7288
        "c2502539d4eb413da194fae615d6e6bf": {
7289
          "model_module": "@jupyter-widgets/base",
7290
          "model_name": "LayoutModel",
7291
          "model_module_version": "1.2.0",
7292
          "state": {
7293
            "_model_module": "@jupyter-widgets/base",
7294
            "_model_module_version": "1.2.0",
7295
            "_model_name": "LayoutModel",
7296
            "_view_count": null,
7297
            "_view_module": "@jupyter-widgets/base",
7298
            "_view_module_version": "1.2.0",
7299
            "_view_name": "LayoutView",
7300
            "align_content": null,
7301
            "align_items": null,
7302
            "align_self": null,
7303
            "border": null,
7304
            "bottom": null,
7305
            "display": null,
7306
            "flex": null,
7307
            "flex_flow": null,
7308
            "grid_area": null,
7309
            "grid_auto_columns": null,
7310
            "grid_auto_flow": null,
7311
            "grid_auto_rows": null,
7312
            "grid_column": null,
7313
            "grid_gap": null,
7314
            "grid_row": null,
7315
            "grid_template_areas": null,
7316
            "grid_template_columns": null,
7317
            "grid_template_rows": null,
7318
            "height": null,
7319
            "justify_content": null,
7320
            "justify_items": null,
7321
            "left": null,
7322
            "margin": null,
7323
            "max_height": null,
7324
            "max_width": null,
7325
            "min_height": null,
7326
            "min_width": null,
7327
            "object_fit": null,
7328
            "object_position": null,
7329
            "order": null,
7330
            "overflow": null,
7331
            "overflow_x": null,
7332
            "overflow_y": null,
7333
            "padding": null,
7334
            "right": null,
7335
            "top": null,
7336
            "visibility": null,
7337
            "width": null
7338
          }
7339
        },
7340
        "95ab78e1349d466d90df3390c0a6ceff": {
7341
          "model_module": "@jupyter-widgets/controls",
7342
          "model_name": "ProgressStyleModel",
7343
          "model_module_version": "1.5.0",
7344
          "state": {
7345
            "_model_module": "@jupyter-widgets/controls",
7346
            "_model_module_version": "1.5.0",
7347
            "_model_name": "ProgressStyleModel",
7348
            "_view_count": null,
7349
            "_view_module": "@jupyter-widgets/base",
7350
            "_view_module_version": "1.2.0",
7351
            "_view_name": "StyleView",
7352
            "bar_color": null,
7353
            "description_width": ""
7354
          }
7355
        },
7356
        "9ca4b24b9dbb4b5a9af5897c46069c49": {
7357
          "model_module": "@jupyter-widgets/base",
7358
          "model_name": "LayoutModel",
7359
          "model_module_version": "1.2.0",
7360
          "state": {
7361
            "_model_module": "@jupyter-widgets/base",
7362
            "_model_module_version": "1.2.0",
7363
            "_model_name": "LayoutModel",
7364
            "_view_count": null,
7365
            "_view_module": "@jupyter-widgets/base",
7366
            "_view_module_version": "1.2.0",
7367
            "_view_name": "LayoutView",
7368
            "align_content": null,
7369
            "align_items": null,
7370
            "align_self": null,
7371
            "border": null,
7372
            "bottom": null,
7373
            "display": null,
7374
            "flex": null,
7375
            "flex_flow": null,
7376
            "grid_area": null,
7377
            "grid_auto_columns": null,
7378
            "grid_auto_flow": null,
7379
            "grid_auto_rows": null,
7380
            "grid_column": null,
7381
            "grid_gap": null,
7382
            "grid_row": null,
7383
            "grid_template_areas": null,
7384
            "grid_template_columns": null,
7385
            "grid_template_rows": null,
7386
            "height": null,
7387
            "justify_content": null,
7388
            "justify_items": null,
7389
            "left": null,
7390
            "margin": null,
7391
            "max_height": null,
7392
            "max_width": null,
7393
            "min_height": null,
7394
            "min_width": null,
7395
            "object_fit": null,
7396
            "object_position": null,
7397
            "order": null,
7398
            "overflow": null,
7399
            "overflow_x": null,
7400
            "overflow_y": null,
7401
            "padding": null,
7402
            "right": null,
7403
            "top": null,
7404
            "visibility": null,
7405
            "width": null
7406
          }
7407
        },
7408
        "5e5449f0eda34f29903b59ca07ac189f": {
7409
          "model_module": "@jupyter-widgets/controls",
7410
          "model_name": "DescriptionStyleModel",
7411
          "model_module_version": "1.5.0",
7412
          "state": {
7413
            "_model_module": "@jupyter-widgets/controls",
7414
            "_model_module_version": "1.5.0",
7415
            "_model_name": "DescriptionStyleModel",
7416
            "_view_count": null,
7417
            "_view_module": "@jupyter-widgets/base",
7418
            "_view_module_version": "1.2.0",
7419
            "_view_name": "StyleView",
7420
            "description_width": ""
7421
          }
7422
        },
7423
        "cae6be4ca33c48acb53cfcad9cbad985": {
7424
          "model_module": "@jupyter-widgets/controls",
7425
          "model_name": "HBoxModel",
7426
          "model_module_version": "1.5.0",
7427
          "state": {
7428
            "_dom_classes": [],
7429
            "_model_module": "@jupyter-widgets/controls",
7430
            "_model_module_version": "1.5.0",
7431
            "_model_name": "HBoxModel",
7432
            "_view_count": null,
7433
            "_view_module": "@jupyter-widgets/controls",
7434
            "_view_module_version": "1.5.0",
7435
            "_view_name": "HBoxView",
7436
            "box_style": "",
7437
            "children": [
7438
              "IPY_MODEL_985bd9a745764ceab05a8e665eefe591",
7439
              "IPY_MODEL_548672800a5b4e5ca72df6d63e01f9d3",
7440
              "IPY_MODEL_62b898efd2ed4810bc8bc7de4e7feebf"
7441
            ],
7442
            "layout": "IPY_MODEL_8de7a61d323e4914901a7cf62a916a07"
7443
          }
7444
        },
7445
        "985bd9a745764ceab05a8e665eefe591": {
7446
          "model_module": "@jupyter-widgets/controls",
7447
          "model_name": "HTMLModel",
7448
          "model_module_version": "1.5.0",
7449
          "state": {
7450
            "_dom_classes": [],
7451
            "_model_module": "@jupyter-widgets/controls",
7452
            "_model_module_version": "1.5.0",
7453
            "_model_name": "HTMLModel",
7454
            "_view_count": null,
7455
            "_view_module": "@jupyter-widgets/controls",
7456
            "_view_module_version": "1.5.0",
7457
            "_view_name": "HTMLView",
7458
            "description": "",
7459
            "description_tooltip": null,
7460
            "layout": "IPY_MODEL_9ee95e5cee6d444588116346f6acedf2",
7461
            "placeholder": "​",
7462
            "style": "IPY_MODEL_9baed57a94ed47e493fa23eb65c2526b",
7463
            "value": "Downloading (…)cial_tokens_map.json: 100%"
7464
          }
7465
        },
7466
        "548672800a5b4e5ca72df6d63e01f9d3": {
7467
          "model_module": "@jupyter-widgets/controls",
7468
          "model_name": "FloatProgressModel",
7469
          "model_module_version": "1.5.0",
7470
          "state": {
7471
            "_dom_classes": [],
7472
            "_model_module": "@jupyter-widgets/controls",
7473
            "_model_module_version": "1.5.0",
7474
            "_model_name": "FloatProgressModel",
7475
            "_view_count": null,
7476
            "_view_module": "@jupyter-widgets/controls",
7477
            "_view_module_version": "1.5.0",
7478
            "_view_name": "ProgressView",
7479
            "bar_style": "success",
7480
            "description": "",
7481
            "description_tooltip": null,
7482
            "layout": "IPY_MODEL_1f3a418ebb47456ea041d7b5b3cf85d5",
7483
            "max": 99,
7484
            "min": 0,
7485
            "orientation": "horizontal",
7486
            "style": "IPY_MODEL_fc0ff589460d47e5b97f745b748dd136",
7487
            "value": 99
7488
          }
7489
        },
7490
        "62b898efd2ed4810bc8bc7de4e7feebf": {
7491
          "model_module": "@jupyter-widgets/controls",
7492
          "model_name": "HTMLModel",
7493
          "model_module_version": "1.5.0",
7494
          "state": {
7495
            "_dom_classes": [],
7496
            "_model_module": "@jupyter-widgets/controls",
7497
            "_model_module_version": "1.5.0",
7498
            "_model_name": "HTMLModel",
7499
            "_view_count": null,
7500
            "_view_module": "@jupyter-widgets/controls",
7501
            "_view_module_version": "1.5.0",
7502
            "_view_name": "HTMLView",
7503
            "description": "",
7504
            "description_tooltip": null,
7505
            "layout": "IPY_MODEL_b55ef28dff8b4b14b5b2cbf0438f8e44",
7506
            "placeholder": "​",
7507
            "style": "IPY_MODEL_8be7a69bd51b448da2111a21822b7482",
7508
            "value": " 99.0/99.0 [00:00&lt;00:00, 7.34kB/s]"
7509
          }
7510
        },
7511
        "8de7a61d323e4914901a7cf62a916a07": {
7512
          "model_module": "@jupyter-widgets/base",
7513
          "model_name": "LayoutModel",
7514
          "model_module_version": "1.2.0",
7515
          "state": {
7516
            "_model_module": "@jupyter-widgets/base",
7517
            "_model_module_version": "1.2.0",
7518
            "_model_name": "LayoutModel",
7519
            "_view_count": null,
7520
            "_view_module": "@jupyter-widgets/base",
7521
            "_view_module_version": "1.2.0",
7522
            "_view_name": "LayoutView",
7523
            "align_content": null,
7524
            "align_items": null,
7525
            "align_self": null,
7526
            "border": null,
7527
            "bottom": null,
7528
            "display": null,
7529
            "flex": null,
7530
            "flex_flow": null,
7531
            "grid_area": null,
7532
            "grid_auto_columns": null,
7533
            "grid_auto_flow": null,
7534
            "grid_auto_rows": null,
7535
            "grid_column": null,
7536
            "grid_gap": null,
7537
            "grid_row": null,
7538
            "grid_template_areas": null,
7539
            "grid_template_columns": null,
7540
            "grid_template_rows": null,
7541
            "height": null,
7542
            "justify_content": null,
7543
            "justify_items": null,
7544
            "left": null,
7545
            "margin": null,
7546
            "max_height": null,
7547
            "max_width": null,
7548
            "min_height": null,
7549
            "min_width": null,
7550
            "object_fit": null,
7551
            "object_position": null,
7552
            "order": null,
7553
            "overflow": null,
7554
            "overflow_x": null,
7555
            "overflow_y": null,
7556
            "padding": null,
7557
            "right": null,
7558
            "top": null,
7559
            "visibility": null,
7560
            "width": null
7561
          }
7562
        },
7563
        "9ee95e5cee6d444588116346f6acedf2": {
7564
          "model_module": "@jupyter-widgets/base",
7565
          "model_name": "LayoutModel",
7566
          "model_module_version": "1.2.0",
7567
          "state": {
7568
            "_model_module": "@jupyter-widgets/base",
7569
            "_model_module_version": "1.2.0",
7570
            "_model_name": "LayoutModel",
7571
            "_view_count": null,
7572
            "_view_module": "@jupyter-widgets/base",
7573
            "_view_module_version": "1.2.0",
7574
            "_view_name": "LayoutView",
7575
            "align_content": null,
7576
            "align_items": null,
7577
            "align_self": null,
7578
            "border": null,
7579
            "bottom": null,
7580
            "display": null,
7581
            "flex": null,
7582
            "flex_flow": null,
7583
            "grid_area": null,
7584
            "grid_auto_columns": null,
7585
            "grid_auto_flow": null,
7586
            "grid_auto_rows": null,
7587
            "grid_column": null,
7588
            "grid_gap": null,
7589
            "grid_row": null,
7590
            "grid_template_areas": null,
7591
            "grid_template_columns": null,
7592
            "grid_template_rows": null,
7593
            "height": null,
7594
            "justify_content": null,
7595
            "justify_items": null,
7596
            "left": null,
7597
            "margin": null,
7598
            "max_height": null,
7599
            "max_width": null,
7600
            "min_height": null,
7601
            "min_width": null,
7602
            "object_fit": null,
7603
            "object_position": null,
7604
            "order": null,
7605
            "overflow": null,
7606
            "overflow_x": null,
7607
            "overflow_y": null,
7608
            "padding": null,
7609
            "right": null,
7610
            "top": null,
7611
            "visibility": null,
7612
            "width": null
7613
          }
7614
        },
7615
        "9baed57a94ed47e493fa23eb65c2526b": {
7616
          "model_module": "@jupyter-widgets/controls",
7617
          "model_name": "DescriptionStyleModel",
7618
          "model_module_version": "1.5.0",
7619
          "state": {
7620
            "_model_module": "@jupyter-widgets/controls",
7621
            "_model_module_version": "1.5.0",
7622
            "_model_name": "DescriptionStyleModel",
7623
            "_view_count": null,
7624
            "_view_module": "@jupyter-widgets/base",
7625
            "_view_module_version": "1.2.0",
7626
            "_view_name": "StyleView",
7627
            "description_width": ""
7628
          }
7629
        },
7630
        "1f3a418ebb47456ea041d7b5b3cf85d5": {
7631
          "model_module": "@jupyter-widgets/base",
7632
          "model_name": "LayoutModel",
7633
          "model_module_version": "1.2.0",
7634
          "state": {
7635
            "_model_module": "@jupyter-widgets/base",
7636
            "_model_module_version": "1.2.0",
7637
            "_model_name": "LayoutModel",
7638
            "_view_count": null,
7639
            "_view_module": "@jupyter-widgets/base",
7640
            "_view_module_version": "1.2.0",
7641
            "_view_name": "LayoutView",
7642
            "align_content": null,
7643
            "align_items": null,
7644
            "align_self": null,
7645
            "border": null,
7646
            "bottom": null,
7647
            "display": null,
7648
            "flex": null,
7649
            "flex_flow": null,
7650
            "grid_area": null,
7651
            "grid_auto_columns": null,
7652
            "grid_auto_flow": null,
7653
            "grid_auto_rows": null,
7654
            "grid_column": null,
7655
            "grid_gap": null,
7656
            "grid_row": null,
7657
            "grid_template_areas": null,
7658
            "grid_template_columns": null,
7659
            "grid_template_rows": null,
7660
            "height": null,
7661
            "justify_content": null,
7662
            "justify_items": null,
7663
            "left": null,
7664
            "margin": null,
7665
            "max_height": null,
7666
            "max_width": null,
7667
            "min_height": null,
7668
            "min_width": null,
7669
            "object_fit": null,
7670
            "object_position": null,
7671
            "order": null,
7672
            "overflow": null,
7673
            "overflow_x": null,
7674
            "overflow_y": null,
7675
            "padding": null,
7676
            "right": null,
7677
            "top": null,
7678
            "visibility": null,
7679
            "width": null
7680
          }
7681
        },
7682
        "fc0ff589460d47e5b97f745b748dd136": {
7683
          "model_module": "@jupyter-widgets/controls",
7684
          "model_name": "ProgressStyleModel",
7685
          "model_module_version": "1.5.0",
7686
          "state": {
7687
            "_model_module": "@jupyter-widgets/controls",
7688
            "_model_module_version": "1.5.0",
7689
            "_model_name": "ProgressStyleModel",
7690
            "_view_count": null,
7691
            "_view_module": "@jupyter-widgets/base",
7692
            "_view_module_version": "1.2.0",
7693
            "_view_name": "StyleView",
7694
            "bar_color": null,
7695
            "description_width": ""
7696
          }
7697
        },
7698
        "b55ef28dff8b4b14b5b2cbf0438f8e44": {
7699
          "model_module": "@jupyter-widgets/base",
7700
          "model_name": "LayoutModel",
7701
          "model_module_version": "1.2.0",
7702
          "state": {
7703
            "_model_module": "@jupyter-widgets/base",
7704
            "_model_module_version": "1.2.0",
7705
            "_model_name": "LayoutModel",
7706
            "_view_count": null,
7707
            "_view_module": "@jupyter-widgets/base",
7708
            "_view_module_version": "1.2.0",
7709
            "_view_name": "LayoutView",
7710
            "align_content": null,
7711
            "align_items": null,
7712
            "align_self": null,
7713
            "border": null,
7714
            "bottom": null,
7715
            "display": null,
7716
            "flex": null,
7717
            "flex_flow": null,
7718
            "grid_area": null,
7719
            "grid_auto_columns": null,
7720
            "grid_auto_flow": null,
7721
            "grid_auto_rows": null,
7722
            "grid_column": null,
7723
            "grid_gap": null,
7724
            "grid_row": null,
7725
            "grid_template_areas": null,
7726
            "grid_template_columns": null,
7727
            "grid_template_rows": null,
7728
            "height": null,
7729
            "justify_content": null,
7730
            "justify_items": null,
7731
            "left": null,
7732
            "margin": null,
7733
            "max_height": null,
7734
            "max_width": null,
7735
            "min_height": null,
7736
            "min_width": null,
7737
            "object_fit": null,
7738
            "object_position": null,
7739
            "order": null,
7740
            "overflow": null,
7741
            "overflow_x": null,
7742
            "overflow_y": null,
7743
            "padding": null,
7744
            "right": null,
7745
            "top": null,
7746
            "visibility": null,
7747
            "width": null
7748
          }
7749
        },
7750
        "8be7a69bd51b448da2111a21822b7482": {
7751
          "model_module": "@jupyter-widgets/controls",
7752
          "model_name": "DescriptionStyleModel",
7753
          "model_module_version": "1.5.0",
7754
          "state": {
7755
            "_model_module": "@jupyter-widgets/controls",
7756
            "_model_module_version": "1.5.0",
7757
            "_model_name": "DescriptionStyleModel",
7758
            "_view_count": null,
7759
            "_view_module": "@jupyter-widgets/base",
7760
            "_view_module_version": "1.2.0",
7761
            "_view_name": "StyleView",
7762
            "description_width": ""
7763
          }
7764
        },
7765
        "18d7c57661f54a2f906b86e5b92a356d": {
7766
          "model_module": "@jupyter-widgets/controls",
7767
          "model_name": "HBoxModel",
7768
          "model_module_version": "1.5.0",
7769
          "state": {
7770
            "_dom_classes": [],
7771
            "_model_module": "@jupyter-widgets/controls",
7772
            "_model_module_version": "1.5.0",
7773
            "_model_name": "HBoxModel",
7774
            "_view_count": null,
7775
            "_view_module": "@jupyter-widgets/controls",
7776
            "_view_module_version": "1.5.0",
7777
            "_view_name": "HBoxView",
7778
            "box_style": "",
7779
            "children": [
7780
              "IPY_MODEL_e132669f57024ebcadc1f0d3899fbda2",
7781
              "IPY_MODEL_f3e78fd1331b4463a243475e12bd2480",
7782
              "IPY_MODEL_386c7bc116b849aaa2bd4c8b0c27b0e9"
7783
            ],
7784
            "layout": "IPY_MODEL_4385c5f8fc5d4d179e02ebf65aae4e0f"
7785
          }
7786
        },
7787
        "e132669f57024ebcadc1f0d3899fbda2": {
7788
          "model_module": "@jupyter-widgets/controls",
7789
          "model_name": "HTMLModel",
7790
          "model_module_version": "1.5.0",
7791
          "state": {
7792
            "_dom_classes": [],
7793
            "_model_module": "@jupyter-widgets/controls",
7794
            "_model_module_version": "1.5.0",
7795
            "_model_name": "HTMLModel",
7796
            "_view_count": null,
7797
            "_view_module": "@jupyter-widgets/controls",
7798
            "_view_module_version": "1.5.0",
7799
            "_view_name": "HTMLView",
7800
            "description": "",
7801
            "description_tooltip": null,
7802
            "layout": "IPY_MODEL_59032aea122142da8744876e7d561bca",
7803
            "placeholder": "​",
7804
            "style": "IPY_MODEL_b7bf29e09c554482823942f17fb09217",
7805
            "value": "Downloading (…)/adapter_config.json: 100%"
7806
          }
7807
        },
7808
        "f3e78fd1331b4463a243475e12bd2480": {
7809
          "model_module": "@jupyter-widgets/controls",
7810
          "model_name": "FloatProgressModel",
7811
          "model_module_version": "1.5.0",
7812
          "state": {
7813
            "_dom_classes": [],
7814
            "_model_module": "@jupyter-widgets/controls",
7815
            "_model_module_version": "1.5.0",
7816
            "_model_name": "FloatProgressModel",
7817
            "_view_count": null,
7818
            "_view_module": "@jupyter-widgets/controls",
7819
            "_view_module_version": "1.5.0",
7820
            "_view_name": "ProgressView",
7821
            "bar_style": "success",
7822
            "description": "",
7823
            "description_tooltip": null,
7824
            "layout": "IPY_MODEL_faf28ec7a6304a1d8e52952aff97c6b4",
7825
            "max": 440,
7826
            "min": 0,
7827
            "orientation": "horizontal",
7828
            "style": "IPY_MODEL_f5ed4018421f446da4ff34741a166d38",
7829
            "value": 440
7830
          }
7831
        },
7832
        "386c7bc116b849aaa2bd4c8b0c27b0e9": {
7833
          "model_module": "@jupyter-widgets/controls",
7834
          "model_name": "HTMLModel",
7835
          "model_module_version": "1.5.0",
7836
          "state": {
7837
            "_dom_classes": [],
7838
            "_model_module": "@jupyter-widgets/controls",
7839
            "_model_module_version": "1.5.0",
7840
            "_model_name": "HTMLModel",
7841
            "_view_count": null,
7842
            "_view_module": "@jupyter-widgets/controls",
7843
            "_view_module_version": "1.5.0",
7844
            "_view_name": "HTMLView",
7845
            "description": "",
7846
            "description_tooltip": null,
7847
            "layout": "IPY_MODEL_1919c4500c88418dad25ed0801033031",
7848
            "placeholder": "​",
7849
            "style": "IPY_MODEL_c4a29910b33d4a348ceef13239cf835e",
7850
            "value": " 440/440 [00:00&lt;00:00, 29.5kB/s]"
7851
          }
7852
        },
7853
        "4385c5f8fc5d4d179e02ebf65aae4e0f": {
7854
          "model_module": "@jupyter-widgets/base",
7855
          "model_name": "LayoutModel",
7856
          "model_module_version": "1.2.0",
7857
          "state": {
7858
            "_model_module": "@jupyter-widgets/base",
7859
            "_model_module_version": "1.2.0",
7860
            "_model_name": "LayoutModel",
7861
            "_view_count": null,
7862
            "_view_module": "@jupyter-widgets/base",
7863
            "_view_module_version": "1.2.0",
7864
            "_view_name": "LayoutView",
7865
            "align_content": null,
7866
            "align_items": null,
7867
            "align_self": null,
7868
            "border": null,
7869
            "bottom": null,
7870
            "display": null,
7871
            "flex": null,
7872
            "flex_flow": null,
7873
            "grid_area": null,
7874
            "grid_auto_columns": null,
7875
            "grid_auto_flow": null,
7876
            "grid_auto_rows": null,
7877
            "grid_column": null,
7878
            "grid_gap": null,
7879
            "grid_row": null,
7880
            "grid_template_areas": null,
7881
            "grid_template_columns": null,
7882
            "grid_template_rows": null,
7883
            "height": null,
7884
            "justify_content": null,
7885
            "justify_items": null,
7886
            "left": null,
7887
            "margin": null,
7888
            "max_height": null,
7889
            "max_width": null,
7890
            "min_height": null,
7891
            "min_width": null,
7892
            "object_fit": null,
7893
            "object_position": null,
7894
            "order": null,
7895
            "overflow": null,
7896
            "overflow_x": null,
7897
            "overflow_y": null,
7898
            "padding": null,
7899
            "right": null,
7900
            "top": null,
7901
            "visibility": null,
7902
            "width": null
7903
          }
7904
        },
7905
        "59032aea122142da8744876e7d561bca": {
7906
          "model_module": "@jupyter-widgets/base",
7907
          "model_name": "LayoutModel",
7908
          "model_module_version": "1.2.0",
7909
          "state": {
7910
            "_model_module": "@jupyter-widgets/base",
7911
            "_model_module_version": "1.2.0",
7912
            "_model_name": "LayoutModel",
7913
            "_view_count": null,
7914
            "_view_module": "@jupyter-widgets/base",
7915
            "_view_module_version": "1.2.0",
7916
            "_view_name": "LayoutView",
7917
            "align_content": null,
7918
            "align_items": null,
7919
            "align_self": null,
7920
            "border": null,
7921
            "bottom": null,
7922
            "display": null,
7923
            "flex": null,
7924
            "flex_flow": null,
7925
            "grid_area": null,
7926
            "grid_auto_columns": null,
7927
            "grid_auto_flow": null,
7928
            "grid_auto_rows": null,
7929
            "grid_column": null,
7930
            "grid_gap": null,
7931
            "grid_row": null,
7932
            "grid_template_areas": null,
7933
            "grid_template_columns": null,
7934
            "grid_template_rows": null,
7935
            "height": null,
7936
            "justify_content": null,
7937
            "justify_items": null,
7938
            "left": null,
7939
            "margin": null,
7940
            "max_height": null,
7941
            "max_width": null,
7942
            "min_height": null,
7943
            "min_width": null,
7944
            "object_fit": null,
7945
            "object_position": null,
7946
            "order": null,
7947
            "overflow": null,
7948
            "overflow_x": null,
7949
            "overflow_y": null,
7950
            "padding": null,
7951
            "right": null,
7952
            "top": null,
7953
            "visibility": null,
7954
            "width": null
7955
          }
7956
        },
7957
        "b7bf29e09c554482823942f17fb09217": {
7958
          "model_module": "@jupyter-widgets/controls",
7959
          "model_name": "DescriptionStyleModel",
7960
          "model_module_version": "1.5.0",
7961
          "state": {
7962
            "_model_module": "@jupyter-widgets/controls",
7963
            "_model_module_version": "1.5.0",
7964
            "_model_name": "DescriptionStyleModel",
7965
            "_view_count": null,
7966
            "_view_module": "@jupyter-widgets/base",
7967
            "_view_module_version": "1.2.0",
7968
            "_view_name": "StyleView",
7969
            "description_width": ""
7970
          }
7971
        },
7972
        "faf28ec7a6304a1d8e52952aff97c6b4": {
7973
          "model_module": "@jupyter-widgets/base",
7974
          "model_name": "LayoutModel",
7975
          "model_module_version": "1.2.0",
7976
          "state": {
7977
            "_model_module": "@jupyter-widgets/base",
7978
            "_model_module_version": "1.2.0",
7979
            "_model_name": "LayoutModel",
7980
            "_view_count": null,
7981
            "_view_module": "@jupyter-widgets/base",
7982
            "_view_module_version": "1.2.0",
7983
            "_view_name": "LayoutView",
7984
            "align_content": null,
7985
            "align_items": null,
7986
            "align_self": null,
7987
            "border": null,
7988
            "bottom": null,
7989
            "display": null,
7990
            "flex": null,
7991
            "flex_flow": null,
7992
            "grid_area": null,
7993
            "grid_auto_columns": null,
7994
            "grid_auto_flow": null,
7995
            "grid_auto_rows": null,
7996
            "grid_column": null,
7997
            "grid_gap": null,
7998
            "grid_row": null,
7999
            "grid_template_areas": null,
8000
            "grid_template_columns": null,
8001
            "grid_template_rows": null,
8002
            "height": null,
8003
            "justify_content": null,
8004
            "justify_items": null,
8005
            "left": null,
8006
            "margin": null,
8007
            "max_height": null,
8008
            "max_width": null,
8009
            "min_height": null,
8010
            "min_width": null,
8011
            "object_fit": null,
8012
            "object_position": null,
8013
            "order": null,
8014
            "overflow": null,
8015
            "overflow_x": null,
8016
            "overflow_y": null,
8017
            "padding": null,
8018
            "right": null,
8019
            "top": null,
8020
            "visibility": null,
8021
            "width": null
8022
          }
8023
        },
8024
        "f5ed4018421f446da4ff34741a166d38": {
8025
          "model_module": "@jupyter-widgets/controls",
8026
          "model_name": "ProgressStyleModel",
8027
          "model_module_version": "1.5.0",
8028
          "state": {
8029
            "_model_module": "@jupyter-widgets/controls",
8030
            "_model_module_version": "1.5.0",
8031
            "_model_name": "ProgressStyleModel",
8032
            "_view_count": null,
8033
            "_view_module": "@jupyter-widgets/base",
8034
            "_view_module_version": "1.2.0",
8035
            "_view_name": "StyleView",
8036
            "bar_color": null,
8037
            "description_width": ""
8038
          }
8039
        },
8040
        "1919c4500c88418dad25ed0801033031": {
8041
          "model_module": "@jupyter-widgets/base",
8042
          "model_name": "LayoutModel",
8043
          "model_module_version": "1.2.0",
8044
          "state": {
8045
            "_model_module": "@jupyter-widgets/base",
8046
            "_model_module_version": "1.2.0",
8047
            "_model_name": "LayoutModel",
8048
            "_view_count": null,
8049
            "_view_module": "@jupyter-widgets/base",
8050
            "_view_module_version": "1.2.0",
8051
            "_view_name": "LayoutView",
8052
            "align_content": null,
8053
            "align_items": null,
8054
            "align_self": null,
8055
            "border": null,
8056
            "bottom": null,
8057
            "display": null,
8058
            "flex": null,
8059
            "flex_flow": null,
8060
            "grid_area": null,
8061
            "grid_auto_columns": null,
8062
            "grid_auto_flow": null,
8063
            "grid_auto_rows": null,
8064
            "grid_column": null,
8065
            "grid_gap": null,
8066
            "grid_row": null,
8067
            "grid_template_areas": null,
8068
            "grid_template_columns": null,
8069
            "grid_template_rows": null,
8070
            "height": null,
8071
            "justify_content": null,
8072
            "justify_items": null,
8073
            "left": null,
8074
            "margin": null,
8075
            "max_height": null,
8076
            "max_width": null,
8077
            "min_height": null,
8078
            "min_width": null,
8079
            "object_fit": null,
8080
            "object_position": null,
8081
            "order": null,
8082
            "overflow": null,
8083
            "overflow_x": null,
8084
            "overflow_y": null,
8085
            "padding": null,
8086
            "right": null,
8087
            "top": null,
8088
            "visibility": null,
8089
            "width": null
8090
          }
8091
        },
8092
        "c4a29910b33d4a348ceef13239cf835e": {
8093
          "model_module": "@jupyter-widgets/controls",
8094
          "model_name": "DescriptionStyleModel",
8095
          "model_module_version": "1.5.0",
8096
          "state": {
8097
            "_model_module": "@jupyter-widgets/controls",
8098
            "_model_module_version": "1.5.0",
8099
            "_model_name": "DescriptionStyleModel",
8100
            "_view_count": null,
8101
            "_view_module": "@jupyter-widgets/base",
8102
            "_view_module_version": "1.2.0",
8103
            "_view_name": "StyleView",
8104
            "description_width": ""
8105
          }
8106
        },
8107
        "31405710033848009cf8227a7611864f": {
8108
          "model_module": "@jupyter-widgets/controls",
8109
          "model_name": "HBoxModel",
8110
          "model_module_version": "1.5.0",
8111
          "state": {
8112
            "_dom_classes": [],
8113
            "_model_module": "@jupyter-widgets/controls",
8114
            "_model_module_version": "1.5.0",
8115
            "_model_name": "HBoxModel",
8116
            "_view_count": null,
8117
            "_view_module": "@jupyter-widgets/controls",
8118
            "_view_module_version": "1.5.0",
8119
            "_view_name": "HBoxView",
8120
            "box_style": "",
8121
            "children": [
8122
              "IPY_MODEL_cc862dc1f8884513b8eefb569ab50424",
8123
              "IPY_MODEL_3cd7c95a507d459f86b9eadd7cfdcb85",
8124
              "IPY_MODEL_683da23b78b54a8eb3c1984cd29bbeb2"
8125
            ],
8126
            "layout": "IPY_MODEL_f022a8aac72d490fb89495f01c929885"
8127
          }
8128
        },
8129
        "cc862dc1f8884513b8eefb569ab50424": {
8130
          "model_module": "@jupyter-widgets/controls",
8131
          "model_name": "HTMLModel",
8132
          "model_module_version": "1.5.0",
8133
          "state": {
8134
            "_dom_classes": [],
8135
            "_model_module": "@jupyter-widgets/controls",
8136
            "_model_module_version": "1.5.0",
8137
            "_model_name": "HTMLModel",
8138
            "_view_count": null,
8139
            "_view_module": "@jupyter-widgets/controls",
8140
            "_view_module_version": "1.5.0",
8141
            "_view_name": "HTMLView",
8142
            "description": "",
8143
            "description_tooltip": null,
8144
            "layout": "IPY_MODEL_ac31ba38940c445e9305623046dc2473",
8145
            "placeholder": "​",
8146
            "style": "IPY_MODEL_1880da50605c435da03377a6e535d3a6",
8147
            "value": "Downloading adapter_model.bin: 100%"
8148
          }
8149
        },
8150
        "3cd7c95a507d459f86b9eadd7cfdcb85": {
8151
          "model_module": "@jupyter-widgets/controls",
8152
          "model_name": "FloatProgressModel",
8153
          "model_module_version": "1.5.0",
8154
          "state": {
8155
            "_dom_classes": [],
8156
            "_model_module": "@jupyter-widgets/controls",
8157
            "_model_module_version": "1.5.0",
8158
            "_model_name": "FloatProgressModel",
8159
            "_view_count": null,
8160
            "_view_module": "@jupyter-widgets/controls",
8161
            "_view_module_version": "1.5.0",
8162
            "_view_name": "ProgressView",
8163
            "bar_style": "success",
8164
            "description": "",
8165
            "description_tooltip": null,
8166
            "layout": "IPY_MODEL_d645078ee2094f37a1bfbaa6727474bb",
8167
            "max": 18907665,
8168
            "min": 0,
8169
            "orientation": "horizontal",
8170
            "style": "IPY_MODEL_a0c4211bdd9e4857b16fbad1d7b1d3e1",
8171
            "value": 18907665
8172
          }
8173
        },
8174
        "683da23b78b54a8eb3c1984cd29bbeb2": {
8175
          "model_module": "@jupyter-widgets/controls",
8176
          "model_name": "HTMLModel",
8177
          "model_module_version": "1.5.0",
8178
          "state": {
8179
            "_dom_classes": [],
8180
            "_model_module": "@jupyter-widgets/controls",
8181
            "_model_module_version": "1.5.0",
8182
            "_model_name": "HTMLModel",
8183
            "_view_count": null,
8184
            "_view_module": "@jupyter-widgets/controls",
8185
            "_view_module_version": "1.5.0",
8186
            "_view_name": "HTMLView",
8187
            "description": "",
8188
            "description_tooltip": null,
8189
            "layout": "IPY_MODEL_3bee676ed0274dee9160f3260f253acf",
8190
            "placeholder": "​",
8191
            "style": "IPY_MODEL_93909168ffc24b5ba6c03a2a067b99c1",
8192
            "value": " 18.9M/18.9M [00:00&lt;00:00, 45.8MB/s]"
8193
          }
8194
        },
8195
        "f022a8aac72d490fb89495f01c929885": {
8196
          "model_module": "@jupyter-widgets/base",
8197
          "model_name": "LayoutModel",
8198
          "model_module_version": "1.2.0",
8199
          "state": {
8200
            "_model_module": "@jupyter-widgets/base",
8201
            "_model_module_version": "1.2.0",
8202
            "_model_name": "LayoutModel",
8203
            "_view_count": null,
8204
            "_view_module": "@jupyter-widgets/base",
8205
            "_view_module_version": "1.2.0",
8206
            "_view_name": "LayoutView",
8207
            "align_content": null,
8208
            "align_items": null,
8209
            "align_self": null,
8210
            "border": null,
8211
            "bottom": null,
8212
            "display": null,
8213
            "flex": null,
8214
            "flex_flow": null,
8215
            "grid_area": null,
8216
            "grid_auto_columns": null,
8217
            "grid_auto_flow": null,
8218
            "grid_auto_rows": null,
8219
            "grid_column": null,
8220
            "grid_gap": null,
8221
            "grid_row": null,
8222
            "grid_template_areas": null,
8223
            "grid_template_columns": null,
8224
            "grid_template_rows": null,
8225
            "height": null,
8226
            "justify_content": null,
8227
            "justify_items": null,
8228
            "left": null,
8229
            "margin": null,
8230
            "max_height": null,
8231
            "max_width": null,
8232
            "min_height": null,
8233
            "min_width": null,
8234
            "object_fit": null,
8235
            "object_position": null,
8236
            "order": null,
8237
            "overflow": null,
8238
            "overflow_x": null,
8239
            "overflow_y": null,
8240
            "padding": null,
8241
            "right": null,
8242
            "top": null,
8243
            "visibility": null,
8244
            "width": null
8245
          }
8246
        },
8247
        "ac31ba38940c445e9305623046dc2473": {
8248
          "model_module": "@jupyter-widgets/base",
8249
          "model_name": "LayoutModel",
8250
          "model_module_version": "1.2.0",
8251
          "state": {
8252
            "_model_module": "@jupyter-widgets/base",
8253
            "_model_module_version": "1.2.0",
8254
            "_model_name": "LayoutModel",
8255
            "_view_count": null,
8256
            "_view_module": "@jupyter-widgets/base",
8257
            "_view_module_version": "1.2.0",
8258
            "_view_name": "LayoutView",
8259
            "align_content": null,
8260
            "align_items": null,
8261
            "align_self": null,
8262
            "border": null,
8263
            "bottom": null,
8264
            "display": null,
8265
            "flex": null,
8266
            "flex_flow": null,
8267
            "grid_area": null,
8268
            "grid_auto_columns": null,
8269
            "grid_auto_flow": null,
8270
            "grid_auto_rows": null,
8271
            "grid_column": null,
8272
            "grid_gap": null,
8273
            "grid_row": null,
8274
            "grid_template_areas": null,
8275
            "grid_template_columns": null,
8276
            "grid_template_rows": null,
8277
            "height": null,
8278
            "justify_content": null,
8279
            "justify_items": null,
8280
            "left": null,
8281
            "margin": null,
8282
            "max_height": null,
8283
            "max_width": null,
8284
            "min_height": null,
8285
            "min_width": null,
8286
            "object_fit": null,
8287
            "object_position": null,
8288
            "order": null,
8289
            "overflow": null,
8290
            "overflow_x": null,
8291
            "overflow_y": null,
8292
            "padding": null,
8293
            "right": null,
8294
            "top": null,
8295
            "visibility": null,
8296
            "width": null
8297
          }
8298
        },
8299
        "1880da50605c435da03377a6e535d3a6": {
8300
          "model_module": "@jupyter-widgets/controls",
8301
          "model_name": "DescriptionStyleModel",
8302
          "model_module_version": "1.5.0",
8303
          "state": {
8304
            "_model_module": "@jupyter-widgets/controls",
8305
            "_model_module_version": "1.5.0",
8306
            "_model_name": "DescriptionStyleModel",
8307
            "_view_count": null,
8308
            "_view_module": "@jupyter-widgets/base",
8309
            "_view_module_version": "1.2.0",
8310
            "_view_name": "StyleView",
8311
            "description_width": ""
8312
          }
8313
        },
8314
        "d645078ee2094f37a1bfbaa6727474bb": {
8315
          "model_module": "@jupyter-widgets/base",
8316
          "model_name": "LayoutModel",
8317
          "model_module_version": "1.2.0",
8318
          "state": {
8319
            "_model_module": "@jupyter-widgets/base",
8320
            "_model_module_version": "1.2.0",
8321
            "_model_name": "LayoutModel",
8322
            "_view_count": null,
8323
            "_view_module": "@jupyter-widgets/base",
8324
            "_view_module_version": "1.2.0",
8325
            "_view_name": "LayoutView",
8326
            "align_content": null,
8327
            "align_items": null,
8328
            "align_self": null,
8329
            "border": null,
8330
            "bottom": null,
8331
            "display": null,
8332
            "flex": null,
8333
            "flex_flow": null,
8334
            "grid_area": null,
8335
            "grid_auto_columns": null,
8336
            "grid_auto_flow": null,
8337
            "grid_auto_rows": null,
8338
            "grid_column": null,
8339
            "grid_gap": null,
8340
            "grid_row": null,
8341
            "grid_template_areas": null,
8342
            "grid_template_columns": null,
8343
            "grid_template_rows": null,
8344
            "height": null,
8345
            "justify_content": null,
8346
            "justify_items": null,
8347
            "left": null,
8348
            "margin": null,
8349
            "max_height": null,
8350
            "max_width": null,
8351
            "min_height": null,
8352
            "min_width": null,
8353
            "object_fit": null,
8354
            "object_position": null,
8355
            "order": null,
8356
            "overflow": null,
8357
            "overflow_x": null,
8358
            "overflow_y": null,
8359
            "padding": null,
8360
            "right": null,
8361
            "top": null,
8362
            "visibility": null,
8363
            "width": null
8364
          }
8365
        },
8366
        "a0c4211bdd9e4857b16fbad1d7b1d3e1": {
8367
          "model_module": "@jupyter-widgets/controls",
8368
          "model_name": "ProgressStyleModel",
8369
          "model_module_version": "1.5.0",
8370
          "state": {
8371
            "_model_module": "@jupyter-widgets/controls",
8372
            "_model_module_version": "1.5.0",
8373
            "_model_name": "ProgressStyleModel",
8374
            "_view_count": null,
8375
            "_view_module": "@jupyter-widgets/base",
8376
            "_view_module_version": "1.2.0",
8377
            "_view_name": "StyleView",
8378
            "bar_color": null,
8379
            "description_width": ""
8380
          }
8381
        },
8382
        "3bee676ed0274dee9160f3260f253acf": {
8383
          "model_module": "@jupyter-widgets/base",
8384
          "model_name": "LayoutModel",
8385
          "model_module_version": "1.2.0",
8386
          "state": {
8387
            "_model_module": "@jupyter-widgets/base",
8388
            "_model_module_version": "1.2.0",
8389
            "_model_name": "LayoutModel",
8390
            "_view_count": null,
8391
            "_view_module": "@jupyter-widgets/base",
8392
            "_view_module_version": "1.2.0",
8393
            "_view_name": "LayoutView",
8394
            "align_content": null,
8395
            "align_items": null,
8396
            "align_self": null,
8397
            "border": null,
8398
            "bottom": null,
8399
            "display": null,
8400
            "flex": null,
8401
            "flex_flow": null,
8402
            "grid_area": null,
8403
            "grid_auto_columns": null,
8404
            "grid_auto_flow": null,
8405
            "grid_auto_rows": null,
8406
            "grid_column": null,
8407
            "grid_gap": null,
8408
            "grid_row": null,
8409
            "grid_template_areas": null,
8410
            "grid_template_columns": null,
8411
            "grid_template_rows": null,
8412
            "height": null,
8413
            "justify_content": null,
8414
            "justify_items": null,
8415
            "left": null,
8416
            "margin": null,
8417
            "max_height": null,
8418
            "max_width": null,
8419
            "min_height": null,
8420
            "min_width": null,
8421
            "object_fit": null,
8422
            "object_position": null,
8423
            "order": null,
8424
            "overflow": null,
8425
            "overflow_x": null,
8426
            "overflow_y": null,
8427
            "padding": null,
8428
            "right": null,
8429
            "top": null,
8430
            "visibility": null,
8431
            "width": null
8432
          }
8433
        },
8434
        "93909168ffc24b5ba6c03a2a067b99c1": {
8435
          "model_module": "@jupyter-widgets/controls",
8436
          "model_name": "DescriptionStyleModel",
8437
          "model_module_version": "1.5.0",
8438
          "state": {
8439
            "_model_module": "@jupyter-widgets/controls",
8440
            "_model_module_version": "1.5.0",
8441
            "_model_name": "DescriptionStyleModel",
8442
            "_view_count": null,
8443
            "_view_module": "@jupyter-widgets/base",
8444
            "_view_module_version": "1.2.0",
8445
            "_view_name": "StyleView",
8446
            "description_width": ""
8447
          }
8448
        }
8449
      }
8450
    }
8451
  },
8452
  "nbformat": 4,
8453
  "nbformat_minor": 0
8454
}