src/main/java/Normalization.java - master - essannikov/java.SearchForDuplicates

src/main/java/Normalization.java
437 строк14 KB
essannikov
Version 1.0
30 янв 2024, 18:07
30 янв 2024, 18:0752ef1fe
import jdk.nashorn.internal.objects.StringIterator;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Array;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;

import static java.lang.Math.round;
import static jdk.nashorn.internal.objects.NativeMath.max;
import static me.xdrop.fuzzywuzzy.FuzzySearch.*;
import static me.xdrop.fuzzywuzzy.FuzzySearch.tokenSetRatio;
import static me.xdrop.fuzzywuzzy.algorithms.WeightedRatio.*;

public class Normalization
{
    public static final String fileNameResult1 = "_RESULT_1.CSV";
    public static final String fileNameResult2 = "_RESULT_2.CSV";
    public static final String stepProcess1 = "1";
    public static final String stepProcess2 = "2";

    public static final String cmdHelp = "help";
    public static final String cmdHelpFile = "src/main/resources/help.txt";

    protected String fileData;
    protected String stepProcess;
    protected int matchValue;
    protected String fileRegex;

    protected List<RegExType> regexList;
    protected List<DataType> dataList;

    protected Messages msg;

    public Normalization() {
        fileData = "";
        stepProcess = "";
        matchValue = 0;
        fileRegex = "";
        regexList = new ArrayList<>();
        dataList = new ArrayList<>();

        msg = Messages.getInstance();
    }

    public boolean exec (String[] args) {
        //Args parse
        try {
            if (!argsParse(args)){
                return true;
            }
        } catch (Exception e) {
            msg.print(msg.typeErr, "msgErrParams");
            return false;
        }

        //Args check
        try {
            argsCheckStep();
        } catch (Exception e) {
            msg.print(msg.typeErr, "msgErrParams");
            return false;
        }
        try {
            argsCheckRegex();
        } catch (Exception e) {
            msg.print(msg.typeErr, "msgErrFileRegex");
            return false;
        }
        try {
            argsCheckData();
        } catch (Exception e) {
            msg.print(msg.typeErr, "msgErrFileData");
            return false;
        }

        //Process
        try {
            processParse();
        } catch (Exception e) {
            msg.print(msg.typeErr, "msgErrProcData");
            return false;
        }

        msg.print(msg.typeOut, "msgGoodSaveFile");
        return true;
    }

    protected void printHelp(){
        StringBuilder builder = new StringBuilder();
        List<String> lines = null;

        try {
            lines = Files.readAllLines(Paths.get(cmdHelpFile));
            lines.forEach(line -> builder.append(line + "\n"));
            System.out.println(builder.toString());
        } catch (IOException e) {
            msg.print(msg.typeOut, "msgErrHelp");
        }
    }

    protected boolean argsParse(String[] args) throws Exception{
        boolean retVal = true;

        switch (args.length){
            case 1:
                String cmdText = args[0].toLowerCase().trim();
                switch (cmdText){
                    case cmdHelp:
                        printHelp();
                        retVal = false;
                        break;
                    default:
                        throw new Exception();
                }
                break;

            case 2:
                fileData = args[0].trim();
                stepProcess = args[1].trim();
                if (stepProcess != stepProcess2){
                    throw new Exception();
                }
                break;

            case 4:
                fileData = args[0].trim();
                stepProcess = args[1].trim();
                matchValue = Integer.parseInt(args[2].trim());
                fileRegex = args[3].trim();
                if (stepProcess != stepProcess1){
                    throw new Exception();
                }
                break;

            default:
                throw new Exception();

        }

        return retVal;
    }

    protected void argsCheckStep() throws Exception{
        if (stepProcess != stepProcess1 &&
                stepProcess != stepProcess2){
            throw new Exception();
        }
    }

    protected void argsCheckRegex() throws Exception{
        RegExType regexListLine;

        if (stepProcess != stepProcess1){
            return;
        }

        String fileCSV = fileRegex;
        try {
            //Add regex
            List<String> lines = Files.readAllLines(Paths.get(fileCSV));

            for (String line : lines) {
                String[] fragments = line.split(";");
                if (fragments.length < 3){
                    //|Position|RegEx|NameCharacteristic...
                    continue;
                }

                int setPos = Integer.parseInt(fragments[0]);
                if (setPos < 1){
                    throw new Exception();
                }

                regexListLine = new RegExType();
                regexListLine.setPosition(fragments[0]);
                regexListLine.setRegex(fragments[1]);
                regexListLine.setName(fragments[2]);
                if (fragments.length > 3){regexListLine.setObligatory(fragments[3]);}
                if (fragments.length > 4){regexListLine.setPair(fragments[4]);}
                if (fragments.length > 5){regexListLine.setMany(fragments[5]);}
                if (fragments.length > 6){regexListLine.setNote(fragments[6]);}

                regexList.add(regexListLine);
            }

            int maxPos = Integer.parseInt(regexList.stream().
                    max(Comparator.comparing(RegExType::getPosition)).
                    get().getPosition());

            if (maxPos != regexList.size()){
                throw new Exception();
            }
        } catch (IOException e) {
            throw new Exception();
        }
    }

    protected void argsCheckData() throws Exception{
        String fileCSV = fileData;

        if (!Files.exists(Path.of(fileCSV))){
            throw new Exception();
        }
    }

    protected void processParse() throws Exception{
        switch (stepProcess){
            case stepProcess1:
                processStepProcess1();
                break;

            case stepProcess2:
                processStepProcess2();
                break;

            default:
                throw new Exception();

        }

        //Save to file
        saveToFile();
    }

    protected void processStepProcess1() throws Exception{
        DataType dataListLine;

        if (stepProcess != stepProcess1){
            throw new Exception();
        }

        String fileCSV = fileData;
        try {
            //Parse
            List<String> lines = Files.readAllLines(Paths.get(fileCSV));

            for (String line : lines) {
                //Get data in columns
                String[] fragments = line.split(";");
                if (fragments.length < 1){
                    continue;
                }

                String maktx = fragments[0];
                //maktx = maktx.replaceAll("\\.",". ");

                fragments = null;
                fragments = maktx.split("\\s+");

                dataListLine = new DataType();
                dataListLine.setMaktx(maktx);
                dataListLine.setData(processTemplate(fragments));

                dataList.add(dataListLine);
            }

            //Search for duplicates
            for (int i = 0; i < dataList.size(); i++) {
                if (!dataList.get(i).getGroupCommon().isEmpty()){
                    continue;
                }

                dataList.get(i).setGroupCommon(String.valueOf(i));

                for (int j = 0; j < dataList.size(); j++) {
                    if (j == i){
                        continue;
                    }
                    if (!dataList.get(j).getGroupCommon().isEmpty()){
                        continue;
                    }

                    //Compare maktx
                    if (compare(dataList.get(i).getMaktx(), dataList.get(j).getMaktx()) > matchValue){
                        dataList.get(j).setGroupCommon(String.valueOf(i));
                    }
                }
            }

        } catch (IOException e) {
            throw new Exception();
        }
    }

    protected void processStepProcess2() throws Exception{
        DataType dataListLine;

        if (stepProcess != stepProcess2){
            throw new Exception();
        }

        String fileCSV = fileData;
        try {
            //Add maktx
            List<String> lines = Files.readAllLines(Paths.get(fileCSV));

            for (String line : lines) {
                //Get data in columns
                String[] fragments = line.split(";");
                if (fragments.length < 3){
                    //|Maktx|GroupCommon|MaktxNew|...
                    continue;
                }

                dataListLine = new DataType();

                dataListLine.setMaktx(fragments[0]);
                dataListLine.setGroupCommon(fragments[1]);
                String maktxNew = "";
                for (int i = 3; i < fragments.length; i++) {
                    maktxNew = maktxNew + " " + fragments[i];
                }
                maktxNew = maktxNew.substring(0,1).toUpperCase() + maktxNew.substring(1);
                dataListLine.setMaktxNew(maktxNew);
                dataListLine.setData(List.of(Arrays.copyOfRange(fragments, 3, fragments.length)));

                dataList.add(dataListLine);
            }
        } catch (IOException e) {
            throw new Exception();
        }
    }

    protected List<String> processTemplate(String[] fragments) throws Exception{
        int maxPos = Integer.parseInt(regexList.stream().
                        max(Comparator.comparing(RegExType::getPosition)).
                        get().getPosition());

        String[] valTmp = new String[maxPos];

        for (int i = 0; i < regexList.size(); i++) {
            String regex = regexList.get(i).getRegex();
            String note = regexList.get(i).getNote();
            String value = "";

            for (int j = 0; j < fragments.length; j++) {
                if (fragments[j] != null && fragments[j].matches(regex)){
                    if (regexList.get(i).getPair().compareTo("X") != 0) {
                        if (value != ""){
                            value = value + " ";
                        }
                        value = value + fragments[j];
                    }
                    fragments[j] = null;

                    if (regexList.get(i).getPair().compareTo("") != 0) {
                        if (j < (fragments.length - 1) && fragments[j + 1] != null){
                            j++;

                            if (value != ""){
                                value = value + " ";
                            }
                            value = value + fragments[j];

                            fragments[j] = null;
                        }
                    }

                    if (regexList.get(i).getMany().compareTo("") == 0){
                        break;
                    }
                }
            }

            if (value != "" && note != ""){
                value = note + " " + value;
            }

            valTmp[Integer.parseInt(regexList.get(i).getPosition()) - 1] = value;
        }

        return Arrays.stream(valTmp).collect(Collectors.toList());
    }

    protected void saveToFile() throws Exception{
        String newFileName = "";

        switch (stepProcess){
            case stepProcess1:
                newFileName = fileData + fileNameResult1;
                break;

            case stepProcess2:
                newFileName = fileData + fileNameResult2;
                break;

            default:
                throw new Exception();

        }

        File newFile = new File(newFileName);
        newFile.createNewFile();

        Files.writeString(Paths.get(newFile.toURI()),
                dataList.stream().map(Object::toString).collect(Collectors.joining(System.lineSeparator())),
                Charset.forName("UTF-8"));
    }

    protected int compare(String s1, String s2){
        int len1 = s1.length();
        int len2 = s2.length();

        if (len1 == 0 || len2 == 0) { return 0; }

        boolean tryPartials = TRY_PARTIALS;
        double unbaseScale = UNBASE_SCALE;
        double partialScale = PARTIAL_SCALE;

        int base = ratio(s1, s2);
        double lenRatio = ((double) Math.max(len1, len2)) / Math.min(len1, len2);

        // if strings are similar length don't use partials
        if (lenRatio < 1.5) tryPartials = false;

        // if one string is much shorter than the other
        if (lenRatio > 8) partialScale = 0.6;

        //weightedRatio() "???

        if (tryPartials) {
            double partial = partialRatio(s1, s2) * partialScale;
            double partialSor = tokenSortPartialRatio(s1, s2) * unbaseScale * partialScale;
            double partialSet = tokenSetPartialRatio(s1, s2) * unbaseScale * partialScale;
            return (int) round(Math.max(base, Math.max(partial, Math.max(partialSor, partialSet))));
        } else {
            double tokenSort = tokenSortRatio(s1, s2) * unbaseScale;
            double tokenSet = tokenSetRatio(s1, s2) * unbaseScale;
            return (int) round(Math.max(base, Math.max(tokenSort, tokenSet)));
        }
    }
}