common/helpers/TerHelper.php - dev - v.bolshakov/AIEcosystem-Testing

common/helpers/TerHelper.php
288 строк8 KB
Developer
Initial commit
03 авг 2026, 17:43
03 авг 2026, 17:437433917
<?php

namespace common\helpers;

use common\kdf\KDocumentStatus;
use common\models\link\LinkUserMemoQ;
use common\models\memoq\Document;
use common\models\memoq\link\LinkDocumentProject;
use common\models\memoq\Project;
use common\models\memoq\TerSync;
use common\models\memoq\XliffSegment;
use common\models\ter\DocumentTer;
use common\models\ter\ProjectTer;
use common\models\ter\UserTer;
use common\shell\Commands\Builder;
use common\shell\Runners\Exec;
use common\shell\Runners\ResultCode;
use common\shell\Runners\ReturnValue;
use common\shell\Runners\StandardError;
use common\shell\Runners\StandardOut;
use Yii;
use yii\helpers\Json;
use yii\helpers\VarDumper;

/**
 * Расширение функций для работы с TER
 *
 * @author Dmitry E. Semenov <sde.tomsk@gmail.com>
 * @copyright Self (c) 2026
 */
class TerHelper
{
    /**
     * Пересчёт статистики TER
     * @param $document_id
     * @return void
     */
    public static function updateTer($document_id)
    {
        $document_id = (int)$document_id;

        self::computationEngineFast($document_id);

        $segment_list = XliffSegment::find()
            ->select([
                '_id',
                'human_final',
                'pre_translation',
                'ai_provider',
                'word_count',
                'ter'
            ])
            ->andWhere([
                'document_id' => $document_id
            ])
            ->andWhere(['>', 'word_count', 0])
            ->all();

        $cnt = 0;
        $sum = 0;

        $ai_cnt = 0;
        $ai_sum = 0;

        /** @var XliffSegment $segment */
        foreach ($segment_list as $segment) {

            if (str_starts_with($segment->ai_provider, 'MT')) {
                // Суммируем общее количество слов
                $ai_cnt += $segment->word_count;
                // Суммируем произведение TER на количество слов
                $ai_sum += ($segment->ter * $segment->word_count);
            }

            // Суммируем общее количество слов
            $cnt += $segment->word_count;
            // Суммируем произведение TER на количество слов
            $sum += ($segment->ter * $segment->word_count);
        }

        if ($cnt > 0) {
            $d_ter = round(($sum / $cnt), 2);
        } else {
            $d_ter = 0.0;
        }

        if ($ai_cnt > 0) {
            $ai_d_ter = round(($ai_sum / $ai_cnt), 2);
        } else {
            $ai_d_ter = 0.0;
        }

        Yii::debug(VarDumper::dumpAsString([
            'sum'      => $sum,
            'cnt'      => $cnt,
            'd_ter'    => $d_ter,
            'ai_sum'   => $ai_sum,
            'ai_cnt'   => $ai_cnt,
            'ai_d_ter' => $ai_d_ter,
        ]));

        $a = Project::getAlias();
        /** @var Project $project */
        $project = Project::find()
            ->alias($a)
            ->innerJoin(['l' => LinkDocumentProject::tableName()], $a . '.id = l.project_id')
            ->andWhere([
                'document_id' => $document_id
            ])
            ->one();

        // по документу
        $document_ter = DocumentTer::findOrNew([
            'document_id' => $document_id
        ]);
        $document_ter->setAttributes([
            'ter'           => $d_ter,
            'ai_ter'        => $ai_d_ter,
            'total_word'    => $cnt,
            'ai_total_word' => $ai_cnt
        ]);
        $document_ter->save();

        // среднее по всем документам проекта
        $a = DocumentTer::getAlias();
        $avg_ter = DocumentTer::find()
            ->select([
                'ter'           => 'sum(ter * total_word) / sum(total_word)',
                'ai_ter'        => 'sum(ai_ter * ai_total_word) / sum(ai_total_word)',
                'total_word'    => 'sum(total_word)',
                'ai_total_word' => 'sum(ai_total_word)',
            ])
            ->alias($a)
            ->innerJoin(['p' => LinkDocumentProject::tableName()], $a . '.document_id = p.document_id')
            ->andWhere([
                'p.project_id' => $project->id
            ])
            ->groupBy('p.project_id')
            ->asArray()
            ->one();

        // по проекту
        $p_ter = ProjectTer::findOrNew([
            'project_id' => $project->id
        ]);

        $p_ter->setAttributes($avg_ter);
        $p_ter->save();

        // по всем проектам пользователя
        $avg_ter = ProjectTer::find()
            ->select([
                'ter'           => 'sum(ter * total_word) / sum(total_word)',
                'ai_ter'        => 'sum(ai_ter * ai_total_word) / sum(ai_total_word)',
                'total_word'    => 'sum(total_word)',
                'ai_total_word' => 'sum(ai_total_word)',
            ])
            ->innerJoin(['p' => Project::tableName()], 'project_id = p.id')
            ->andWhere([
//                'p.status_id'       => [
//                    KDocumentStatus::TRANSLATION_FINISHED,
//                    KDocumentStatus::PROOFREADING_FINISHED
//                ],
                'p.creator_user_id' => $project->creator_user_id
            ])
            ->groupBy('p.creator_user_id')
            ->asArray()
            ->one();

        $user_id = LinkUserMemoQ::find()
            ->select('user_id')
            ->andWhere([
                'm_user_id' => $project->creator_user_id
            ])
            ->scalar();

        // по пользователю
        $user_ter = UserTer::findOrNew([
            'user_id' => $user_id
        ]);

        $user_ter->setAttributes($avg_ter);
        $user_ter->save();

        TerSync::updateAll([
            'last_update' => DateHelper::now(),
            'is_request'  => 0
        ], [
            'document_id' => $document_id
        ]);
    }

    /**
     * Выполнение расчётов
     */
    public static function computationEngineFast($document_id)
    {
        $search = Yii::getAlias('@root/script/ter_document.py');
        $command = new Builder();
        $command
            ->setCommand('python3')
            ->addParam($search)
            ->addParam($document_id);

        $cmd = (string)$command;

        $data = [
            'cmd' => $cmd,
        ];

        $run = new Exec();
        $run->run($command);

        if ($run instanceof ReturnValue) {
            $data['return_value'] = $run->getReturnValue();
        }
        if ($run instanceof ResultCode) {
            $data['return_code'] = $run->getResultCode();
        }
        if ($run instanceof StandardError) {
            $data['standard_error'] = $run->getStandardError();
        }
        if ($run instanceof StandardOut) {
            $data['standard_out'] = $run->getStandardOut();
        }

        Yii::debug(VarDumper::dumpAsString($data));

        return true;
    }

    /**
     * Выполнение расчётов
     */
    public static function computationEngine($reference, $hypothesis)
    {
        $search = Yii::getAlias('@root/script/ter.py');
        $command = new Builder();
        $command
            ->setCommand('python3')
            ->addParam($search)
            ->addParam($reference)
            ->addParam($hypothesis);

        $cmd = (string)$command;

        $data = [
            'cmd' => $cmd,
        ];

        $run = new Exec();
        $run->run($command);

        if ($run instanceof ReturnValue) {
            $data['return_value'] = $run->getReturnValue();
        }
        if ($run instanceof ResultCode) {
            $data['return_code'] = $run->getResultCode();
        }
        if ($run instanceof StandardError) {
            $data['standard_error'] = $run->getStandardError();
        }
        if ($run instanceof StandardOut) {
            $data['standard_out'] = $run->getStandardOut();
        }

//        Yii::debug(VarDumper::dumpAsString($data));

        $json = ArrayHelper::getValue($data, 'return_value.0');
        $ter = null;

        try {
            $body = Json::decode($json);
            $ter = (float)ArrayHelper::getValue($body, '0.ter');
            $is_ok = true;
            $message = 'OK';
        } catch (\Exception $e) {
            $message = $e->getMessage();
            $is_ok = false;
        }

        return [
            'is_ok'   => $is_ok,
            'ter'     => $ter,
            'message' => $message
        ];
    }
}