2
This tutorial is a copy of OpenPrompt's tutorial/1.1_mixed_template.py
3
The only modification is in lines 98 to 102
5
1. OpenPrompt provides pre-processing of data, such as prompt template formatting
6
2. OpenPrompt pre-process the model input, such as: prompt soft embedding
7
3. OpenDelta modify the backbone model, such as: Adapter, Lora, Compactor, etc.
8
4. OpenPrompt post-process the model output, such as: extract logits at <mask> position, apply prompt verbalizer
12
from datasets import load_dataset
13
from datasets import load_from_disk
15
raw_dataset = load_from_disk("/home/hx/huggingface_datasets/saved_to_disk/super_glue.cb")
22
from openprompt.data_utils import InputExample
25
for split in ['train', 'validation', 'test']:
27
for data in raw_dataset[split]:
28
input_example = InputExample(text_a = data['premise'], text_b = data['hypothesis'], label=int(data['label']), guid=data['idx'])
29
dataset[split].append(input_example)
30
print(dataset['train'][0])
33
from openprompt.plms import load_plm
34
plm, tokenizer, model_config, WrapperClass = load_plm("t5", "t5-base")
38
from openprompt.prompts import MixedTemplate
39
template_text = '{"placeholder":"text_a"} {"soft"} {"soft"} {"soft"} {"placeholder":"text_b"}? {"soft"} {"soft"} {"soft"} {"mask"}.'
40
mytemplate = MixedTemplate(model=plm, tokenizer=tokenizer, text=template_text)
44
wrapped_example = mytemplate.wrap_one_example(dataset['train'][0])
53
wrapped_t5tokenizer = WrapperClass(max_seq_length=128, decoder_max_length=3, tokenizer=tokenizer,truncate_method="head")
55
from openprompt.plms import T5TokenizerWrapper
56
wrapped_t5tokenizer= T5TokenizerWrapper(max_seq_length=128, decoder_max_length=3, tokenizer=tokenizer,truncate_method="head")
59
tokenized_example = wrapped_t5tokenizer.tokenize_one_example(wrapped_example, teacher_forcing=False)
60
print(tokenized_example)
61
print(tokenizer.convert_ids_to_tokens(tokenized_example['input_ids']))
62
print(tokenizer.convert_ids_to_tokens(tokenized_example['decoder_input_ids']))
68
for split in ['train', 'validation', 'test']:
69
model_inputs[split] = []
70
for sample in dataset[split]:
71
tokenized_example = wrapped_t5tokenizer.tokenize_one_example(mytemplate.wrap_one_example(sample), teacher_forcing=False)
72
model_inputs[split].append(tokenized_example)
76
from openprompt import PromptDataLoader
78
train_dataloader = PromptDataLoader(dataset=dataset["train"], template=mytemplate, tokenizer=tokenizer,
79
tokenizer_wrapper_class=WrapperClass, max_seq_length=256, decoder_max_length=3,
80
batch_size=4,shuffle=True, teacher_forcing=False, predict_eos_token=False,
81
truncate_method="head")
87
from openprompt.prompts import ManualVerbalizer
91
myverbalizer = ManualVerbalizer(tokenizer, num_classes=3, label_words=[["yes"], ["no"], ["maybe"]])
93
print("label_words_ids", myverbalizer.label_words_ids)
98
from opendelta import LoraModel
100
delta_model = LoraModel(backbone_model=plm, modified_modules=["SelfAttention.q", "SelfAttention.v"])
101
delta_model.freeze_module(exclude=["deltas"], set_state_dict=True)
104
from openprompt import PromptForClassification
107
prompt_model = PromptForClassification(plm=plm, template=mytemplate, verbalizer=myverbalizer)
109
prompt_model = prompt_model.cuda()
112
from transformers import AdamW, get_linear_schedule_with_warmup
113
loss_func = torch.nn.CrossEntropyLoss()
114
no_decay = ['bias', 'LayerNorm.weight']
116
optimizer_grouped_parameters = [
117
{'params': [p for n, p in prompt_model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
118
{'params': [p for n, p in prompt_model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
120
print([n for n, p in prompt_model.named_parameters()])
122
optimizer = AdamW(optimizer_grouped_parameters, lr=1e-4)
124
for epoch in range(30):
126
for step, inputs in enumerate(train_dataloader):
128
inputs = inputs.cuda()
129
logits = prompt_model(inputs)
130
labels = inputs['label']
131
loss = loss_func(logits, labels)
133
tot_loss += loss.item()
135
optimizer.zero_grad()
137
print("Epoch {}, average loss: {}".format(epoch, tot_loss/(step+1)), flush=True)
140
validation_dataloader = PromptDataLoader(dataset=dataset["validation"], template=mytemplate, tokenizer=tokenizer,
141
tokenizer_wrapper_class=WrapperClass, max_seq_length=256, decoder_max_length=3,
142
batch_size=4,shuffle=False, teacher_forcing=False, predict_eos_token=False,
143
truncate_method="head")
147
for step, inputs in enumerate(validation_dataloader):
149
inputs = inputs.cuda()
150
logits = prompt_model(inputs)
151
labels = inputs['label']
152
alllabels.extend(labels.cpu().tolist())
153
allpreds.extend(torch.argmax(logits, dim=-1).cpu().tolist())
155
acc = sum([int(i==j) for i,j in zip(allpreds, alllabels)])/len(allpreds)