HateXplain/Preprocess/dataCollect.py - master - Chriss_Nickell/Course_Project

HateXplain/Preprocess/dataCollect.py
425 строк14 KB
Norman
first_commit
03 июн 2026, 06:09
03 июн 2026, 06:09dd02054
import pandas as pd
from glob import glob
import json
from tqdm import tqdm_notebook,tqdm
from difflib import SequenceMatcher
from collections import Counter
from .preProcess import ek_extra_preprocess
from .attentionCal import aggregate_attention
from .spanMatcher import returnMask,returnMaskonetime
from transformers import BertTokenizer
from .utils import CheckForGreater,most_frequent
from .preProcess import *   
from transformers import BertTokenizer
from os import path
import pickle
import numpy as np
import re

def set_name(params):
    file_name='Data/Total_data'
    if(params['bert_tokens']):
        file_name+='_bert'
    else:
        file_name+='_normal'
    
    file_name+='_'+params['type_attention']+'_'+str(params['variance'])+'_'+str(params['max_length'])
    if(params['decay']):
        file_name+='_'+params['method']+'_'+str(params['window'])+'_'+str(params['alpha']) +'_'+str(params['p_value'])    
    file_name+='_'+str(params['num_classes'])+'.pickle'
    return file_name



def get_annotated_data(params):
    #temp_read = pd.read_pickle(params['data_file'])
    with open(params['data_file'], 'r') as fp:
        data = json.load(fp)
    dict_data=[]
    for key in data:
        temp={}
        temp['post_id']=key
        temp['text']=data[key]['post_tokens']
        final_label=[]
        for i in range(1,4):
            temp['annotatorid'+str(i)]=data[key]['annotators'][i-1]['annotator_id']
#             temp['explain'+str(i)]=data[key]['annotators'][i-1]['rationales']
            temp['target'+str(i)]=data[key]['annotators'][i-1]['target']
            temp['label'+str(i)]=data[key]['annotators'][i-1]['label']
            final_label.append(temp['label'+str(i)])

        final_label_id=max(final_label,key=final_label.count)
        temp['rationales']=data[key]['rationales']
            
        if(params['class_names']=='Data/classes_two.npy'):
            if(final_label.count(final_label_id)==1):
                temp['final_label']='undecided'
            else:
                if(final_label_id in ['hatespeech','offensive']):
                    final_label_id='toxic'
                else:
                    final_label_id='non-toxic'
                temp['final_label']=final_label_id

        
        else:
            if(final_label.count(final_label_id)==1):
                temp['final_label']='undecided'
            else:
                temp['final_label']=final_label_id

        
        
        
        dict_data.append(temp)    
    temp_read = pd.DataFrame(dict_data)  
    return temp_read    





def get_training_data(data,params,tokenizer):
    '''input: data is a dataframe text ids attentions labels column only'''
    '''output: training data in the columns post_id,text, attention and labels '''

    majority=params['majority']
    post_ids_list=[]
    text_list=[]
    attention_list=[]
    label_list=[]
    count=0
    count_confused=0
    print('total_data',len(data))
    for index,row in tqdm(data.iterrows(),total=len(data)):
        #print(params)
        text=row['text']
        post_id=row['post_id']

        annotation_list=[row['label1'],row['label2'],row['label3']] 
        annotation=row['final_label']
        
        if(annotation != 'undecided'):
            tokens_all,attention_masks=returnMask(row,params,tokenizer)
            attention_vector= aggregate_attention(attention_masks,row, params)     
            attention_list.append(attention_vector)
            text_list.append(tokens_all)
            label_list.append(annotation)
            post_ids_list.append(post_id)
        else:
            count_confused+=1
            
    print("attention_error:",count)
    print("no_majority:",count_confused)
    # Calling DataFrame constructor after zipping 
    # both lists, with columns specified 
    training_data = pd.DataFrame(list(zip(post_ids_list,text_list,attention_list,label_list)), 
                   columns =['Post_id','Text', 'Attention' , 'Label']) 
    
    
    filename=set_name(params)
    training_data.to_pickle(filename)
    return training_data

##### Data collection for test data
def get_test_data(data,params,message='text'):
    '''input: data is a dataframe text ids labels column only'''
    '''output: training data in the columns post_id,text (tokens) , attentions (normal) and labels'''
    
    if(params['bert_tokens']):
        print('Loading BERT tokenizer...')
        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=False)
    else:
        tokenizer=None
    
    
    post_ids_list=[]
    text_list=[]
    attention_list=[]
    label_list=[]
    print('total_data',len(data))
    for index,row in tqdm(data.iterrows(),total=len(data)):
        post_id=row['post_id']
        annotation=row['final_label']
        tokens_all,attention_masks=returnMask(row,params,tokenizer)
        attention_vector= aggregate_attention(attention_masks,row, params) 
        attention_list.append(attention_vector)
        text_list.append(tokens_all)
        label_list.append(annotation)
        post_ids_list.append(post_id)
    
    
    # Calling DataFrame constructor after zipping 
    # both lists, with columns specified 
    training_data = pd.DataFrame(list(zip(post_ids_list,text_list,attention_list,label_list)), 
                   columns =['Post_id','Text', 'Attention' , 'Label']) 
    
    
    return training_data







def convert_data(test_data,params,list_dict,rational_present=True,topk=2):
    """this converts the data to be with or without the rationals based on the previous predictions"""
    """input: params -- input dict, list_dict -- previous predictions containing rationals
    rational_present -- whether to keep rational only or remove them only
    topk -- how many words to select"""
    
    temp_dict={}
    for ele in list_dict:
        temp_dict[ele['annotation_id']]=ele['rationales'][0]['soft_rationale_predictions']
    
    test_data_modified=[]
    
    for index,row in tqdm(test_data.iterrows(),total=len(test_data)):
        try:
            attention=temp_dict[row['Post_id']]
        except KeyError:
            continue
        topk_indices = sorted(range(len(attention)), key=lambda i: attention[i])[-topk:]
        new_text =[]
        new_attention =[]
        if(rational_present):
            if(params['bert_tokens']):
                new_attention =[0]
                new_text = [101]
            for i in range(len(row['Text'])):
                if(i in topk_indices):
                    new_text.append(row['Text'][i])
                    new_attention.append(row['Attention'][i])
            if(params['bert_tokens']):
                new_attention.append(0)
                new_text.append(102)
        else:
            for i in range(len(row['Text'])):
                if(i not in topk_indices):
                    new_text.append(row['Text'][i])
                    new_attention.append(row['Attention'][i])
        test_data_modified.append([row['Post_id'],new_text,new_attention,row['Label']])

    df=pd.DataFrame(test_data_modified,columns=test_data.columns)
    return df



def transform_dummy_data(sentences):
    post_id_list=['temp']*len(sentences)
    pred_list=['normal']*len(sentences)
    explanation_list=[]
    sentences_list=[]
    for i in range(len(sentences)):
        explanation_list.append([])
        sentences_list.append(sentences[i].split(" "))
    df=pd.DataFrame(list(zip(post_id_list,sentences_list,pred_list,pred_list,
                             pred_list,explanation_list,pred_list)),
                         columns=['post_id', 'text', 'label1','label2','label3', 'rationales', 'final_label'])
    
    return df


def collect_data(params):
    if(params['bert_tokens']):
        print('Loading BERT tokenizer...')
        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=False)
    else:
        tokenizer=None
    data_all_labelled=get_annotated_data(params)
    train_data=get_training_data(data_all_labelled,params,tokenizer)
    return train_data
























##### ONLY FOR ONE TIME USAGE
def get_training_data_one_time(data,params,tokenizer):
    '''input: data is a dataframe text ids attentions labels column only'''
    '''output: training data in the columns post_id,text, attention and labels '''

    majority=params['majority']
    
    dict_data={}
   
    
    print('total_data',len(data))
    
    for index,row in tqdm(data.iterrows(),total=len(data)):
        post_id=row['post_id']
        dict_data[post_id]={}
        dict_data[post_id]['post_id']=post_id
        dict_data[post_id]['annotators']=[]
        #dict_data[post_id]['old_vs_new']=row['old_vs_new']
        selection_list=[row['explain1'],row['explain2'],row['explain3']]
        tokens_all,attention_masks,string_parts,list_pos,span_list,list_mask \
        =returnMaskonetime(row,params,tokenizer,data_type=row['old_vs_new'])
        temp_users=[]
        
        rational_list=[]
        
        for i in range(1,4):
            temp_user_dict={}
            temp_user_dict['label']=row['pred'+str(i)]
            #temp_user_dict['rationales']=attention_masks[i-1]
            if(row['final_annotation'] not in ['normal','non-toxic','undecided']):
                if(row['old_vs_new']=='new' and i<3):
                    rational_list.append(attention_masks[i-1])
                elif(row['old_vs_new']=='old' and (row['pred'+str(i)] not in ['normal','non-toxic'])):
                    rational_list.append(attention_masks[i-1])

            temp_user_dict['annotator_id']=row['workerid'+str(i)]
            temp_user_dict['target']=row['target'+str(i)]
            temp_users.append(temp_user_dict)
            
            
        dict_data[post_id]['rationales']=rational_list
        dict_data[post_id]['post_tokens']=tokens_all
        dict_data[post_id]['annotators']=temp_users
    
    return dict_data        





#### OLDcode remove at last
def return_inverse_dict():
    with open("../../main/id_orig_seid_Mapping.json") as f:
        id_dict_orig = json.load(f) 



    orig_dict_id={}
    for key in tqdm(id_dict_orig.keys()):    
        orig_text=id_dict_orig[key][0]
        seid_text=id_dict_orig[key][1]

        orig_dict_id[seid_text]=[key,orig_text]
    return orig_dict_id


def return_id_orig(text,orig_dict_id):
    try:
        #to return the test directly 
        return orig_dict_id[text][0],orig_dict_id[text][1]
    except:
        max_sim=0
        max_text=""
        for key in orig_dict_id.keys():
            text_id=orig_dict_id[key][0]
            orig_text=orig_dict_id[key][1]
            sim=similar(key,text)
            if(sim>max_sim):
                max_sim=sim
                max_text=key
            if(sim>0.95):
                return text_id,orig_text
        print(text,"||",max_text,"||",max_sim)
        return -1,-1

    
def get_text_information(df,key,text_id_map):
    dict_text={}
    text='Input.text'
    gender='Answer.Gender'
    miscellanous='Answer.Miscellaneous'
    origin='Answer.Origin'
    race='Answer.Race'
    sexual='Answer.Sexual'
    annotated='Answer.sentiment'
    religion='Answer.Religion'
    selection ='Answer.selections'
    for i in range(1,7):
        if(i>1):
            own_text=text+str(i)
            own_gender=gender+str(i)
            own_miscl=miscellanous+str(i)
            own_origin=origin+str(i)
            own_race=race+str(i)
            own_sexual=sexual+str(i)
            own_annotated=annotated+str(i)
            own_religion=religion+str(i)
            
        else:
            own_text=text
            own_gender=gender
            own_miscl=miscellanous
            own_origin=origin
            own_race=race
            own_sexual=sexual
            own_annotated=annotated
            own_religion=religion
        own_selection = selection+str(i)
        if(df.iloc[0][own_text]==df.iloc[1][own_text]==df.iloc[2][own_text]):
            
            
            id_text,orig_text=return_id_orig(df.iloc[0][own_text],text_id_map)
            if(id_text==-1):
                id_text=key+"_nf_"+str(i)
                orig_text=cleanhtml(df.iloc[0][own_text])
                print(id_text)
            
            dict_text[id_text]={'text':orig_text}
            for k in range(0,3):
                str_user='user'+str(k+1)
                dict_text[str_user]={
                                        'annotation':df.iloc[k][own_annotated],
                                        'gender':df.iloc[k][own_gender],
                                        'miscellanous':df.iloc[k][own_miscl],
                                        'origin':df.iloc[k][own_origin],
                                        'race':df.iloc[k][own_race],
                                        'sexual':df.iloc[k][own_sexual],  
                                        'religion':df.iloc[k][own_religion],
                                        'selection':df.iloc[k][own_selection]
                                    }
            
            
    
    
    return dict_text
    
def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()
    
def get_dict_comments(orig_dict_id,data_all_labelled,params):
    dict_comment_file = 'dict_comments.dat'
    not_rerun=params['not_recollect']
    if(path.isfile(dict_comment_file) and not_rerun):
        with open(dict_comment_file, 'rb') as handle:
            dict_comments = pickle.load(handle)
    else:
        print('hello')
        dict_comments={}
        data_all_labelled=data_all_labelled.reset_index(drop=True) 
        grouped_data=data_all_labelled.groupby("HITId")
        grouped_data=dict(list(grouped_data))
        for key in tqdm(grouped_data.keys()):
            dict_new=get_text_information(grouped_data[key],key,orig_dict_id)
            for key in dict_new.keys():
                dict_comments[key]=dict_new[key]

        with open(dict_comment_file, 'wb') as handle:
            pickle.dump(dict_comments, handle, protocol=pickle.HIGHEST_PROTOCOL)
    return dict_comments