CSS-LM

1
import os
2
import sys
3
import csv
4
from statistics import mean, pstdev
5

6
#dir_input = sys.argv[1]
7
#dir_output = sys.argv[2]
8
#N = sys.argv[3]
9
#mode = sys.argv[4]
10
#if type(N) == int:
11
#    N = str(N)
12

13
#dataset
14
#datasets = {"semeval":"","sst5":"_sst5","scicite":"_scicite","aclintent":"_aclintent","sciie":"_sciie","chemprot":"_chemprot"}
15
datasets = {"semeval":"","scicite":"_scicite","sciie":"_sciie"}
16

17
#datasets = {"semeval":""}
18
#datasets = {"sst5":"_sst5"}
19
#datasets = {"scicite":"_scicite"}
20
#datasets = {"aclintent":"_aclintent"}
21
#datasets = {"sciie":"_sciie"}
22
#datasets = {"chemprot":"_chemprot"}
23

24
#mode
25
modes = ["roberta","bert"]
26
#modes = ["roberta"]
27
#modes = ["bert"]
28

29
#times
30
times_list=list(range(1,6)) #1~5
31
#times_list=list(range(1,3))
32

33
#supervised_data
34
#supervised_data=["16","24","32","48"]
35
#supervised_data=["16"]
36
#supervised_data=["24"]
37
supervised_data=["32"]
38

39

40
for dataset in datasets:
41

42
    #finetune
43
    ###################################
44
    ###################################
45
    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!")
46
    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!")
47
    print("!!!! Dataset:",dataset,"!!!!!")
48
    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!")
49
    print("!!!!!!!!!!!!!!!!!!!!!!!!!!!")
50
    print("#################")
51
    print("Model:","Finetune")
52
    print("#################")
53
    print("-----------------")
54

55
    for mode in modes:
56
        test_i=list()
57
        eval_i=list()
58

59
        if mode == "roberta":
60
            dir_input = "baseline_n_result_open_domain"+str(datasets[dataset])+"_entropy_finetune"
61
            #print(dir_input)
62
            #print("--")
63
        elif mode == "bert":
64
            dir_input = "baseline_bert_n_result_open_domain"+str(datasets[dataset])+"_entropy_finetune"
65
            #print(dir_input)
66
            #print("==")
67

68
        #1~5
69
        for i in times_list: #1~5
70
            file_dir = dir_input+"_"+str(i)
71
            try:
72
                fileDir = os.listdir(file_dir)
73
                #find file
74
                test_file = [f for f in fileDir if "test_paper_results" in f and "txt_no_eval" not in f and supervised_data[0] in f][0]
75
                eval_file = [f for f in fileDir if "eval_results" in f and supervised_data[0] in f][0]
76
                #test
77
                with open(file_dir+"/"+test_file) as f:
78
                    for line in f:
79
                        line = line.strip().split()
80
                        if line[0] == "F1":
81
                            test_i.append(float(line[-1].replace("%","")))
82
                #eval
83
                with open(file_dir+"/"+eval_file) as f:
84
                    for line in f:
85
                        line = line.strip().split()
86
                        if line[0] == "eval_accuracy":
87
                            eval_i.append(float(line[-1])*100)
88
                print("File:", file_dir, ": DONE")
89
            except:
90
                print("File:", file_dir, ": PENDING...")
91
                #print("No file")
92
                #print("Less than 5 times")
93
                pass
94

95
        ####Caculate
96
        print("Mode:",mode)
97
        try:
98
            mean_test_i = mean(test_i)
99
            mean_eval_i = mean(eval_i)
100
            std_test_i = pstdev(test_i)
101
            std_eval_i = pstdev(eval_i)
102
            print("test:","{:.2f}%".format(mean_test_i)," ; ","std:","{:.2f}".format(std_test_i))
103
            print("eval:","{:.2f}%".format(mean_eval_i)," ; ","std:","{:.2f}".format(std_eval_i))
104
            print("test:",test_i)
105
            print("eval:",eval_i)
106
            print("------")
107
        except:
108
            print("test:","Pending...")
109
            print("eval:","Pending...")
110
            print("------")
111
    print("-------------------------------------------")
112
    ###################################
113
    ###################################
114

115

116

117
    #sscl
118
    ###################################
119
    ###################################
120
    #print("Dataset:",dataset)
121
    #print("!!!!!!!!!!!!!!!!!")
122
    print("#################")
123
    print("Model:","sscl")
124
    print("#################")
125
    print("-----------------")
126

127
    for mode in modes:
128
        test_i=list()
129
        eval_i=list()
130

131
        if mode == "roberta":
132
            dir_input = "baseline_n_result_open_domain"+str(datasets[dataset])+"_entropy_sscl"
133
            #print(dir_input)
134
        elif mode == "bert":
135
            dir_input = "baseline_bert_n_result_open_domain"+str(datasets[dataset])+"_entropy_sscl"
136
            #print(dir_input)
137

138
        #1~5
139
        for i in times_list: #1~5
140
            file_dir = dir_input+"_"+str(i)
141
            try:
142
                fileDir = os.listdir(file_dir)
143
                #find file
144
                test_file = [f for f in fileDir if "test_paper_results" in f and "txt_no_eval" not in f and supervised_data[0] in f][0]
145
                eval_file = [f for f in fileDir if "eval_results" in f and supervised_data[0] in f][0]
146
                #test
147
                with open(file_dir+"/"+test_file) as f:
148
                    for line in f:
149
                        line = line.strip().split()
150
                        if line[0] == "F1":
151
                            test_i.append(float(line[-1].replace("%","")))
152
                #eval
153
                with open(file_dir+"/"+eval_file) as f:
154
                    for line in f:
155
                        line = line.strip().split()
156
                        if line[0] == "eval_accuracy":
157
                            eval_i.append(float(line[-1])*100)
158
                print("File:", file_dir, ": DONE")
159
            except:
160
                print("File:", file_dir, ": PENDING...")
161
                #print("No file")
162
                #print("Less than 5 times")
163
                pass
164

165
        ####Caculate
166
        print("Mode:",mode)
167
        try:
168
            mean_test_i = mean(test_i)
169
            mean_eval_i = mean(eval_i)
170
            std_test_i = pstdev(test_i)
171
            std_eval_i = pstdev(eval_i)
172
            print("test:","{:.2f}%".format(mean_test_i)," ; ","std:","{:.2f}".format(std_test_i))
173
            print("eval:","{:.2f}%".format(mean_eval_i)," ; ","std:","{:.2f}".format(std_eval_i))
174
            print("test:",test_i)
175
            print("eval:",eval_i)
176
            print("------")
177
        except:
178
            print("test:","Pending...")
179
            print("eval:","Pending...")
180
            print("------")
181
    print("-------------------------------------------")
182
    ###################################
183
    ###################################
184

185

186

187

188
'''
189
    #st
190
    ###################################
191
    ###################################
192
    #print("Dataset:",dataset)
193
    #print("!!!!!!!!!!!!!!!!!")
194
    print("#################")
195
    print("Model:","st")
196
    print("#################")
197
    print("-----------------")
198

199
    for mode in modes:
200

201
        if mode == "roberta":
202
            dir_input = "retriver_n_result_open_domain"+str(datasets[dataset])+"_entropy_st"
203
            #print(dir_input)
204
        elif mode == "bert":
205
            dir_input = "retriver_bert_n_result_open_domain"+str(datasets[dataset])+"_entropy_st"
206
            #print(dir_input)
207

208
        #1~5
209
        for k in ["16","32","48"]:
210
            test_i=list()
211
            eval_i=list()
212
            for i in times_list: #1~5
213
                file_dir = dir_input+"_"+str(k)+"_"+str(i)
214
                try:
215
                    fileDir = os.listdir(file_dir)
216
                    #find file
217
                    test_file = [f for f in fileDir if "test_paper_results" in f and "txt_no_eval" not in f and supervised_data[0] in f][0]
218
                    eval_file = [f for f in fileDir if "eval_results" in f and supervised_data[0] in f][0]
219
                    #test
220
                    with open(file_dir+"/"+test_file) as f:
221
                        for line in f:
222
                            line = line.strip().split()
223
                            if line[0] == "F1":
224
                                test_i.append(float(line[-1].replace("%","")))
225
                    #eval
226
                    with open(file_dir+"/"+eval_file) as f:
227
                        for line in f:
228
                            line = line.strip().split()
229
                            if line[0] == "eval_accuracy":
230
                                eval_i.append(float(line[-1])*100)
231
                    print("File:", file_dir, ": DONE")
232
                except:
233
                    print("File:", file_dir, ": PENDING...")
234
                    #print("Less than 5 times")
235
                    pass
236

237
            ####Caculate
238
            print("Mode:",mode)
239
            print("K:",k)
240
            try:
241
                mean_test_i = mean(test_i)
242
                mean_eval_i = mean(eval_i)
243
                std_test_i = pstdev(test_i)
244
                std_eval_i = pstdev(eval_i)
245
                print("test:","{:.2f}%".format(mean_test_i)," ; ","std:","{:.2f}".format(std_test_i))
246
                print("eval:","{:.2f}%".format(mean_eval_i)," ; ","std:","{:.2f}".format(std_eval_i))
247
                print("test:",test_i)
248
                print("eval:",eval_i)
249
                print("------")
250
            except:
251
                print("test:","Pending...")
252
                print("eval:","Pending...")
253
                print("------")
254
        print("-------------------------------------------")
255
    print("===========================================")
256
    ###################################
257
    ###################################
258

259

260
    print("===========================================")
261
    print("===========================================")
262

263

264

265
    #sscl_dt
266
    ###################################
267
    ###################################
268
    #print("Dataset:",dataset)
269
    #print("!!!!!!!!!!!!!!!!!")
270
    print("#################")
271
    print("Model:","sscl_dt")
272
    print("#################")
273
    print("-----------------")
274

275
    for mode in modes:
276

277
        if mode == "roberta":
278
            dir_input = "retriver_n_result_open_domain"+str(datasets[dataset])+"_entropy_sscl_dt"
279
            #print(dir_input)
280
        elif mode == "bert":
281
            dir_input = "retriver_bert_n_result_open_domain"+str(datasets[dataset])+"_entropy_sscl_dt"
282
            #print(dir_input)
283

284
        #1~5
285
        for k in ["16","32","48"]:
286
            test_i=list()
287
            eval_i=list()
288
            for i in times_list: #1~5
289
                file_dir = dir_input+"_"+str(k)+"_"+str(i)
290
                try:
291
                    fileDir = os.listdir(file_dir)
292
                    #find file
293
                    test_file = [f for f in fileDir if "test_paper_results" in f and "txt_no_eval" not in f and supervised_data[0] in f][0]
294
                    eval_file = [f for f in fileDir if "eval_results" in f and supervised_data[0] in f][0]
295
                    #test
296
                    with open(file_dir+"/"+test_file) as f:
297
                        for line in f:
298
                            line = line.strip().split()
299
                            if line[0] == "F1":
300
                                test_i.append(float(line[-1].replace("%","")))
301
                    #eval
302
                    with open(file_dir+"/"+eval_file) as f:
303
                        for line in f:
304
                            line = line.strip().split()
305
                            if line[0] == "eval_accuracy":
306
                                eval_i.append(float(line[-1])*100)
307
                    print("File:", file_dir, ": DONE")
308
                except:
309
                    print("File:", file_dir, ": PENDING...")
310
                    #print("Less than 5 times")
311
                    pass
312

313
            ####Caculate
314
            print("Mode:",mode)
315
            print("K:",k)
316
            try:
317
                mean_test_i = mean(test_i)
318
                mean_eval_i = mean(eval_i)
319
                std_test_i = pstdev(test_i)
320
                std_eval_i = pstdev(eval_i)
321
                print("test:","{:.2f}%".format(mean_test_i)," ; ","std:","{:.2f}".format(std_test_i))
322
                print("eval:","{:.2f}%".format(mean_eval_i)," ; ","std:","{:.2f}".format(std_eval_i))
323
                print("test:",test_i)
324
                print("eval:",eval_i)
325
                print("------")
326
            except:
327
                print("test:","Pending...")
328
                print("eval:","Pending...")
329
                print("------")
330
        print("-------------------------------------------")
331
    print("===========================================")
332
    ###################################
333
    ###################################
334

335

336
'''
337