LLM-FineTuning-Large-Language-Models

1
import os
2
import torch
3

4
# You can run the Notebook `Pytorch-Roberta_Large.ipynb`  either Locally or in Kaggle - Just modify the 'ROOT_DIR' variable to properly refer to the dataset
5
# ROOT_DIR = '../input/feedback-prize-2021/' # Kaggle
6
ROOT_DIR = '../input/' # local
7

8
# MODEL_NAME = 'roberta-large'
9
MODEL_NAME = 'roberta-base'
10

11
MODEL_PATH = 'model'
12

13
RUN_NAME = f"{MODEL_NAME}"
14

15
MAX_LEN = 512
16

17
DOC_STRIDE = 128
18

19
config = {'train_batch_size': 4,
20
          'valid_batch_size': 1,
21
          'epochs': 5,
22
          'learning_rates': [2.5e-5, 2.5e-5, 2.5e-6, 2.5e-6, 2.5e-7],
23
          'max_grad_norm': 10,
24
          'device': 'cuda' if torch.cuda.is_available() else 'cpu',
25
          'model_name': MODEL_NAME,
26
          'max_length': MAX_LEN,
27
          'doc_stride': DOC_STRIDE,
28
          }
29

30
# Note in above, I have 5 Learning rates for 5 epochs
31

32
output_labels = ['O', 'B-Lead', 'I-Lead', 'B-Position', 'I-Position', 'B-Claim', 'I-Claim', 'B-Counterclaim', 'I-Counterclaim',
33
          'B-Rebuttal', 'I-Rebuttal', 'B-Evidence', 'I-Evidence', 'B-Concluding Statement', 'I-Concluding Statement']
34

35
LABELS_TO_IDS = {v:k for k,v in enumerate(output_labels)}
36
IDS_TO_LABELS = {k:v for k,v in enumerate(output_labels)}