Prompt-Transferability
332 строки · 9.3 Кб
1# -*- coding: utf-8 -*-
2"""pipeline.ipynb
3
4Automatically generated by Colaboratory.
5
6Original file is located at
7https://colab.research.google.com/drive/1-m2ywJVcfgCHOcEN-4agAbLz7tRGqMvM
8"""
9
10'''准备模型和数据'''
11'''这里模型就用model这个变量'''
12'''数据之后用example作为演示'''
13'''使用的时候替换成自己的model就可以了'''
14#import numpy as np
15import torch
16import config
17#from activate_neuron.mymodel import *
18#import activate_neuron.mymodel as mymodel
19#from activate_neuron.utils import *
20#import activate_neuron.utils as utils
21
22
23#from transformers import AutoConfig, AutoModelForMaskedLM
24#from model.modelling_roberta import RobertaForMaskedLM
25#from reader.reader import init_dataset, init_formatter, init_test_dataset
26
27import argparse
28import os
29import torch
30import logging
31import random
32import numpy as np
33
34from tools.init_tool import init_all
35from config_parser import create_config
36from tools.valid_tool import valid
37from torch.autograd import Variable
38
39logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
40datefmt='%m/%d/%Y %H:%M:%S',
41level=logging.INFO)
42
43logger = logging.getLogger(__name__)
44
45def set_random_seed(seed):
46"""Set random seed for reproducability."""
47
48if seed is not None and seed > 0:
49random.seed(seed)
50np.random.seed(seed)
51torch.manual_seed(seed)
52torch.cuda.manual_seed_all(seed)
53
54
55
56def relu(tmp):
57return 1*(tmp > 0)*tmp
58
59def topk(obj, k):
60M=-10000
61obj = list(obj)[:]
62idlist = []
63for i in range(k):
64idlist.append(obj.index(max(obj)))
65obj[obj.index(max(obj))]=M
66return idlist
67
68def relu(tmp):
69return 1*(tmp > 0)*tmp
70
71def topk(obj, k):
72M=-10000
73obj = list(obj)[:]
74idlist = []
75for i in range(k):
76idlist.append(obj.index(max(obj)))
77obj[obj.index(max(obj))]=M
78return idlist
79
80
81
82
83if __name__ == "__main__":
84
85parser = argparse.ArgumentParser()
86parser.add_argument('--config', '-c', help="specific config file", required=True)
87parser.add_argument('--gpu', '-g', help="gpu id list")
88parser.add_argument('--local_rank', type=int, help='local rank', default=-1)
89parser.add_argument('--do_test', help="do test while training or not", action="store_true")
90parser.add_argument('--checkpoint', help="checkpoint file path", type=str, default=None)
91parser.add_argument('--comment', help="checkpoint file path", default=None)
92parser.add_argument("--seed", type=int, default=None)
93parser.add_argument("--prompt_emb_output", type=bool, default=False)
94parser.add_argument("--save_name", type=str, default=None)
95parser.add_argument("--replacing_prompt", type=str, default=None)
96parser.add_argument("--pre_train_mlm", default=False, action='store_true')
97parser.add_argument("--task_transfer_projector", default=False, action='store_true')
98parser.add_argument("--model_transfer_projector", default=False, action='store_true')
99parser.add_argument("--activate_neuron", default=True, action='store_true')
100parser.add_argument("--mode", type=str, default="valid")
101parser.add_argument("--projector", type=str, default=None)
102
103
104args = parser.parse_args()
105configFilePath = args.config
106
107
108config = create_config(configFilePath)
109
110
111
112use_gpu = True
113gpu_list = []
114if args.gpu is None:
115use_gpu = False
116else:
117use_gpu = True
118os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
119
120device_list = args.gpu.split(",")
121for a in range(0, len(device_list)):
122gpu_list.append(int(a))
123
124os.system("clear")
125config.set('distributed', 'local_rank', args.local_rank)
126config.set("distributed", "use", False)
127if config.getboolean("distributed", "use") and len(gpu_list)>1:
128torch.cuda.set_device(gpu_list[args.local_rank])
129torch.distributed.init_process_group(backend=config.get("distributed", "backend"))
130config.set('distributed', 'gpu_num', len(gpu_list))
131
132cuda = torch.cuda.is_available()
133logger.info("CUDA available: %s" % str(cuda))
134if not cuda and len(gpu_list) > 0:
135logger.error("CUDA is not available but specific gpu id")
136raise NotImplementedError
137set_random_seed(args.seed)
138
139
140########
141'''
142formatter = "mlmPrompt"
143config.set("data","train_formatter_type",formatter)
144config.set("data","valid_formatter_type",formatter)
145config.set("data","test_formatter_type",formatter)
146config.set("model","model_name","mlmPrompt")
147'''
148########
149
150
151
152parameters = init_all(config, gpu_list, args.checkpoint, args.mode, local_rank = args.local_rank, args=args)
153do_test = False
154
155model = parameters["model"]
156valid_dataset = parameters["valid_dataset"]
157
158
159##########################
160##########################
161
162
163'''准备hook'''
164'''这是提取特征的代码'''
165outputs=[[] for _ in range(12)]
166def save_ppt_outputs1_hook(n):
167def fn(_,__,output):
168#print("=====")
169#print(output)
170#print("----")
171#print(output.shape) #torch.Size([1, 1, 3072])
172#print("=====")
173#exit()
174outputs[n].append(output.detach().to("cpu"))
175#outputs[n].append(output.detach())
176return fn
177
178
179for n in range(12):
180#这里面提取feature的模组可以改变,这里因为我自定义模型的原因要两层roberta
181#for l in model.state_dict().keys():
182# print(l)
183#print("====")
184#exit()
185
186#decoder
187model.encoder.decoder.block[n].layer[2].DenseReluDense.wi.register_forward_hook(save_ppt_outputs1_hook(n))
188
189#encoder
190#model.encoder.encoder.block[n].layer[1].DenseReluDense.wi.register_forward_hook(save_ppt_outputs1_hook(n))
191
192
193
194
195
196
197'''将数据通过模型'''
198'''hook会自动将中间层的激活储存在outputs中'''
199model.eval()
200valid(model, parameters["valid_dataset"], 1, None, config, gpu_list, parameters["output_function"], mode=args.mode, args=args)
201
202
203#################################################
204#################################################
205#################################################
206
207
208'''
209print(len(outputs)) #12
210print(len(outputs[0])) #17 epoch
211print(len(outputs[0][0])) #64
212print(len(outputs[0][0][0])) #231
213print(len(outputs[0][0][0][0])) #3072
214#outputs[][][][][] , layer:12, epoch:17, batch_size:64, input_length:231, neuron:3072
215'''
216
217#merge 17 epoch
218for k in range(12):
219#outputs[k] = relu(np.concatenate(outputs[k]))
220#outputs[k] = torch.relu(torch.cat(outputs[k]))
221outputs[k] = torch.cat(outputs[k])
222#print(outputs[k])
223#print(outputs[k].shape)
224#exit()
225
226
227'''
228print(len(outputs)) #12
229print(len(outputs[0])) #17 epoch
230print(len(outputs[0][0])) #64
231print(len(outputs[0][0][0])) #231
232print(len(outputs[0][0][0][0])) #3072
233#outputs[][][][][] , layer:12, epoch:17, batch_size:64, input_length:231, neuron:3072
234'''
235
236
237'''这部分是根据论文里的代码找到某个neuron的最大激活'''
238'''
239#划定层数
240#layer = np.random.randint(12)
241layer = torch.randint(1,12,(1,))
242#决定neuron
243#neuron = np.random.randint(3072)
244neuron = torch.randint(1,3072,(1,))
245#这里面是得到了某层的某个neuron的所有激活
246neuron_activation = outputs[layer][:,:,neuron]
247max_activation = [neuron_activation[i,:length[i]].max() for i in range(size)]
248print(neuron_activation)
249print(max_activation)
250exit()
251'''
252
253
254
255outputs = torch.stack(outputs)
256
257#decoder
258outputs = outputs[:,:1,:1,:] #12 layers, [mask]
259
260#encoder
261#outputs = outputs[:,:,100:101,:] #12 layers, [mask]
262
263#print(outputs.shape)
264# [12, 1, 1, 3072] --> 12, 1(batch_size), (target_length), 3072
265
266# [12, 2, 1, 3072] --> 12, 1(batch_size), (target_length), 3072
267
268
269#print(outputs)
270#print(save_dir)
271#exit()
272
273
274save_name = args.replacing_prompt.strip().split("/")[-1].split(".")[0]
275#print(save_name)
276#exit()
277dir = "task_activated_neuron"
278if os.path.isdir(dir):
279save_dir = dir+"/"+save_name
280if os.path.isdir(save_dir):
281torch.save(outputs,save_dir+"/task_activated_neuron")
282else:
283os.mkdir(save_dir)
284torch.save(outputs,save_dir+"/task_activated_neuron")
285else:
286os.mkdir(dir)
287save_dir = dir+"/"+save_name
288os.mkdir(save_dir)
289torch.save(outputs,save_dir+"/task_activated_neuron")
290
291
292print("==Prompt emb==")
293print(outputs.shape)
294print("Save Done")
295print("==============")
296
297
298
299
300
301
302
303
304
305
306'''
307size = 8 # number of the sentences
308length = 231 #sentence length
309#Activated neuron for a task-specific prompt
310for layer in range(1,12):
311for neuron in range(1,3072):
312neuron_activation = outputs[layer][:,:,neuron]
313print(outputs[layer].shape)
314print(neuron_activation.shape)
315exit()
316max_activation = [neuron_activation[i,:length[i]].max() for i in range(size)]
317print(neuron_activation)
318print("------------")
319print(max_activation)
320print("============")
321exit()
322'''
323
324
325
326'''选择头几个句子展示'''
327'''
328N = 4
329indexes = topk(max_activation,N)
330for ids in indexes:
331print(tokenizer.decode(example['input_ids'][ids,:length[ids]]))
332'''
333