data_preprocessing.py - master - veta_el/brain_language_patterns

data_preprocessing.py
72 строки3 KB
veta_el
upload files
18 апр 2025, 08:13
18 апр 2025, 08:139a1b94b
import pandas as pd
import numpy as np

from sdv.single_table import GaussianCopulaSynthesizer
from sdv.evaluation.single_table import evaluate_quality
from sdv.metadata import SingleTableMetadata

from config import Config


def transform_database ():
    def xlsx_csv (path: str):
        base = pd.DataFrame(pd.read_excel(path))
        base.drop(index=base.index[0], axis=0, inplace=True)
        base.reset_index(drop = True, inplace = True)
        base.columns=['subj', 'chan', 'lang', 'n250', 'n400', 'hem', 'ant', 'cond', 'rel']
        return base

    def to_format (data: pd.DataFrame):
        cols = (Config.ALL_FEATURES).split (';')
        cols.pop ()
        cols.append ('cond')
        cols.append ('rel')
        all_rows = []
        for i in range (0, data.shape [0], 20):
            row_list = []
            for j in range (0, 20):
                row_list.append (data.at [i+j, 'n250'])
                row_list.append (data.at [i+j, 'n400'])
            row_list.append (data.at [i, 'cond'])
            row_list.append (data.at [i, 'rel'])
            all_rows.append (row_list)
        base = pd.DataFrame(all_rows, columns = cols)
        base['rel'] = np.where(base['rel'] == 'related', '1', '0')
        base.reset_index(drop = True, inplace = True)
        return base


    base = xlsx_csv (Config.FOLDER_PATH+Config.FULL_DATA_PATH)
    base = to_format (base)

    pred_v = ['id', 'morph', 'orth', 'sem']
    cols = (Config.ALL_FEATURES).split (';')
    for v in pred_v: # Save data with required conditions
        transformed_base = pd.DataFrame(columns = cols)
        for i in range (0, base.shape [0]):
            if base.at [i, 'cond'] == v:
                transformed_base.loc[len(transformed_base)] = base.loc[i]
        transformed_base = transformed_base.sample(frac = 1).reset_index(drop = True) # Reset index and save
        transformed_base.to_csv (Config.FOLDER_PATH+'//transformed_'+v+'.csv')
    return ['transformed_id.csv', 'transformed_morph.csv', 'transformed_orth.csv', 'transformed_sem.csv']

def data_synth (base_name: str):
    data = pd.read_csv (Config.FOLDER_PATH+'//'+base_name)
    synth_base = data
    for i in range(0, 2): # Synth data
        to_synth = data[data['rel'] == i]
        metadata = SingleTableMetadata()
        metadata.detect_from_dataframe(data = to_synth)
        model = GaussianCopulaSynthesizer(metadata)
        model.fit(to_synth)
        to_return = model.sample(106)
        print (str (evaluate_quality(to_synth, to_return, metadata).get_score())) # Check quality
        synth_base = pd.concat([synth_base, to_return])
    synth_base = synth_base.sample(frac = 1).reset_index(drop = True) # Reset index and train/test split
    train = synth_base.iloc [:int((len(synth_base)+1)/3*2)]
    train.reset_index(drop = True, inplace = True)
    train.to_csv (Config.FOLDER_PATH+'//train_'+base_name)
    test = synth_base.iloc [int((len(synth_base)+1)/3*2):]
    test.reset_index(drop = True, inplace = True)
    test.to_csv (Config.FOLDER_PATH+'//test_'+base_name)
    return train, test