rss_parser.php - master - potoskuevge/123

rss_parser.php
220 строк13 KB
Grirorii Potoskuev
create: ant_index.html, conf.html, config_env_agents.json, config_panel.json, negative_themes.json, positive_themes.json, README parser.txt, README.txt, README_config.txt, result.json, rss_feeds.json, rss_parser.php, SimAnt_config_settings_guide.txt
14 апр 2026, 23:30
Верифицирован
14 апр 2026, 23:301336756
<?php

declare(strict_types=1);
date_default_timezone_set('UTC');

$baseDir = __DIR__;
$feedsFile = $baseDir . DIRECTORY_SEPARATOR . 'rss_feeds.json';
$positiveFile = $baseDir . DIRECTORY_SEPARATOR . 'positive_themes.json';
$negativeFile = $baseDir . DIRECTORY_SEPARATOR . 'negative_themes.json';
$resultFile = $baseDir . DIRECTORY_SEPARATOR . 'result.json';
$stateFile = $baseDir . DIRECTORY_SEPARATOR . 'parser_state.json';

function readJsonFile(string $path): array {
    if (!is_file($path)) throw new RuntimeException('Missing file: ' . $path);
    $content = file_get_contents($path);
    if ($content === false) throw new RuntimeException('Cannot read file: ' . $path);
    $data = json_decode($content, true);
    if (!is_array($data)) throw new RuntimeException('Invalid JSON in file: ' . $path);
    return $data;
}
function writeJsonFile(string $path, array $data, bool $prettyPrint): void {
    $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES;
    if ($prettyPrint) $flags |= JSON_PRETTY_PRINT;
    $json = json_encode($data, $flags);
    if ($json === false) throw new RuntimeException('Cannot encode JSON for file: ' . $path);
    if (file_put_contents($path, $json) === false) throw new RuntimeException('Cannot write file: ' . $path);
}
function readState(string $path): array {
    if (!is_file($path)) return array('food'=>0.0,'beetle'=>0.0,'caterpillar'=>0.0,'moth'=>0.0);
    $data = readJsonFile($path);
    return array(
        'food' => isset($data['food']) ? (float)$data['food'] : 0.0,
        'beetle' => isset($data['beetle']) ? (float)$data['beetle'] : 0.0,
        'caterpillar' => isset($data['caterpillar']) ? (float)$data['caterpillar'] : 0.0,
        'moth' => isset($data['moth']) ? (float)$data['moth'] : 0.0
    );
}
function fetchUrl(string $url, string $userAgent, int $timeout): string {
    if (function_exists('curl_init')) {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
        curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
        curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
        $result = curl_exec($ch);
        if ($result === false) {
            $error = curl_error($ch);
            curl_close($ch);
            throw new RuntimeException('cURL error: ' . $error . ' for ' . $url);
        }
        $httpCode = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);
        if ($httpCode >= 400) throw new RuntimeException('HTTP ' . $httpCode . ' for ' . $url);
        return (string)$result;
    }
    $context = stream_context_create(array(
        'http' => array('timeout' => $timeout, 'user_agent' => $userAgent),
        'ssl' => array('verify_peer' => false, 'verify_peer_name' => false)
    ));
    $result = @file_get_contents($url, false, $context);
    if ($result === false) throw new RuntimeException('file_get_contents failed for ' . $url);
    return $result;
}
function parseRssItems(string $xmlString, int $maxItems): array {
    libxml_use_internal_errors(true);
    $xml = simplexml_load_string($xmlString, 'SimpleXMLElement', LIBXML_NOCDATA);
    if ($xml === false) {
        $errors = libxml_get_errors();
        libxml_clear_errors();
        throw new RuntimeException('Cannot parse XML, errors: ' . count($errors));
    }
    $items = array();
    if (isset($xml->channel->item)) {
        foreach ($xml->channel->item as $item) {
            $items[] = array('title'=>trim((string)$item->title), 'description'=>trim((string)$item->description), 'pubDate'=>trim((string)$item->pubDate), 'link'=>trim((string)$item->link));
            if (count($items) >= $maxItems) break;
        }
    } elseif (isset($xml->entry)) {
        foreach ($xml->entry as $entry) {
            $summary = isset($entry->summary) ? trim((string)$entry->summary) : (isset($entry->content) ? trim((string)$entry->content) : '');
            $link = isset($entry->link['href']) ? trim((string)$entry->link['href']) : '';
            $pubDate = isset($entry->updated) ? trim((string)$entry->updated) : (isset($entry->published) ? trim((string)$entry->published) : '');
            $items[] = array('title'=>trim((string)$entry->title), 'description'=>$summary, 'pubDate'=>$pubDate, 'link'=>$link);
            if (count($items) >= $maxItems) break;
        }
    }
    return $items;
}
function normalizeText(string $text): string {
    $text = mb_strtolower($text, 'UTF-8');
    $text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8');
    $text = strip_tags($text);
    $text = preg_replace('/[\x00-\x1F\x7F]+/u', ' ', $text);
    $text = preg_replace('/[^\p{L}\p{N}\s\-]/u', ' ', $text);
    $text = preg_replace('/\s+/u', ' ', $text);
    return trim((string)$text);
}
function countKeywordHits(string $text, array $keywords): int {
    $hits = 0;
    foreach ($keywords as $keyword) {
        $kw = normalizeText((string)$keyword);
        if ($kw === '') continue;
        $pattern = '/(^|\s)' . preg_quote($kw, '/') . '(\s|$)/u';
        if (preg_match_all($pattern, ' ' . $text . ' ', $m)) $hits += count($m[0]);
    }
    return $hits;
}
function analyzeThemes(string $text, array $themes): array {
    $result = array('theme_hits'=>array(), 'scores'=>array('food'=>0.0,'beetle'=>0.0,'caterpillar'=>0.0,'moth'=>0.0));
    foreach ($themes as $theme) {
        $themeId = isset($theme['id']) ? (string)$theme['id'] : 'unknown';
        $weight = isset($theme['weight']) ? (float)$theme['weight'] : 1.0;
        $keywords = isset($theme['keywords']) && is_array($theme['keywords']) ? $theme['keywords'] : array();
        $mapsTo = isset($theme['maps_to']) && is_array($theme['maps_to']) ? $theme['maps_to'] : array();
        $hits = countKeywordHits($text, $keywords);
        if ($hits <= 0) continue;
        $themeScore = $hits * $weight;
        $result['theme_hits'][$themeId] = array('hits'=>$hits,'weight'=>$weight,'score'=>$themeScore);
        foreach ($mapsTo as $target => $factor) {
            if (!isset($result['scores'][$target])) $result['scores'][$target] = 0.0;
            $result['scores'][$target] += $themeScore * (float)$factor;
        }
    }
    return $result;
}
function smoothValue(float $previous, float $current, float $alpha): float {
    return ($previous * (1.0 - $alpha)) + ($current * $alpha);
}
function clampInt(float $value, int $min, int $max): int {
    $rounded = (int)round($value);
    if ($rounded < $min) return $min;
    if ($rounded > $max) return $max;
    return $rounded;
}

try {
    $feedsConfig = readJsonFile($feedsFile);
    $positiveConfig = readJsonFile($positiveFile);
    $negativeConfig = readJsonFile($negativeFile);
    $lookbackHours = isset($feedsConfig['polling']['lookback_hours']) ? (int)$feedsConfig['polling']['lookback_hours'] : 24;
    $maxItemsPerFeed = isset($feedsConfig['polling']['max_items_per_feed']) ? (int)$feedsConfig['polling']['max_items_per_feed'] : 50;
    $userAgent = isset($feedsConfig['polling']['user_agent']) ? (string)$feedsConfig['polling']['user_agent'] : 'SimAntRSSBot/3.0';
    $timeoutSeconds = isset($feedsConfig['polling']['timeout_seconds']) ? (int)$feedsConfig['polling']['timeout_seconds'] : 12;
    $prettyPrint = !empty($feedsConfig['output']['pretty_print']);
    $foodScale = isset($feedsConfig['mapping']['food_scale']) ? (float)$feedsConfig['mapping']['food_scale'] : 1.0;
    $beetleScale = isset($feedsConfig['mapping']['beetle_scale']) ? (float)$feedsConfig['mapping']['beetle_scale'] : 1.0;
    $caterpillarScale = isset($feedsConfig['mapping']['caterpillar_scale']) ? (float)$feedsConfig['mapping']['caterpillar_scale'] : 1.0;
    $mothScale = isset($feedsConfig['mapping']['moth_scale']) ? (float)$feedsConfig['mapping']['moth_scale'] : 1.0;
    $maxFood = isset($feedsConfig['mapping']['max_food']) ? (int)$feedsConfig['mapping']['max_food'] : 250;
    $maxBeetle = isset($feedsConfig['mapping']['max_beetle']) ? (int)$feedsConfig['mapping']['max_beetle'] : 250;
    $maxCaterpillar = isset($feedsConfig['mapping']['max_caterpillar']) ? (int)$feedsConfig['mapping']['max_caterpillar'] : 250;
    $maxMoth = isset($feedsConfig['mapping']['max_moth']) ? (int)$feedsConfig['mapping']['max_moth'] : 250;
    $alpha = isset($feedsConfig['mapping']['smoothing_alpha']) ? (float)$feedsConfig['mapping']['smoothing_alpha'] : 0.35;
    $positiveThemes = isset($positiveConfig['themes']) && is_array($positiveConfig['themes']) ? $positiveConfig['themes'] : array();
    $negativeThemes = isset($negativeConfig['themes']) && is_array($negativeConfig['themes']) ? $negativeConfig['themes'] : array();
    $cutoffTimestamp = time() - ($lookbackHours * 3600);
    $rawScores = array('food'=>0.0,'beetle'=>0.0,'caterpillar'=>0.0,'moth'=>0.0);
    $feedStats = array(); $themeDebug = array(); $seenLinks = array(); $newsCount = 0;
    foreach ($feedsConfig['feeds'] as $feed) {
        $feedName = isset($feed['name']) ? (string)$feed['name'] : 'unknown';
        $feedUrl = isset($feed['url']) ? (string)$feed['url'] : '';
        if ($feedUrl === '') { $feedStats[] = array('name'=>$feedName,'status'=>'skipped','reason'=>'empty url'); continue; }
        try {
            $xml = fetchUrl($feedUrl, $userAgent, $timeoutSeconds);
            $items = parseRssItems($xml, $maxItemsPerFeed);
            $accepted = 0;
            foreach ($items as $item) {
                $link = isset($item['link']) ? (string)$item['link'] : '';
                if ($link !== '' && isset($seenLinks[$link])) continue;
                $pubDate = isset($item['pubDate']) ? strtotime((string)$item['pubDate']) : false;
                if ($pubDate !== false && $pubDate < $cutoffTimestamp) continue;
                $text = normalizeText((isset($item['title']) ? (string)$item['title'] : '') . ' ' . (isset($item['description']) ? (string)$item['description'] : ''));
                if ($text === '') continue;
                if ($link !== '') $seenLinks[$link] = true;
                $positiveResult = analyzeThemes($text, $positiveThemes);
                $negativeResult = analyzeThemes($text, $negativeThemes);
                foreach ($positiveResult['scores'] as $target => $score) { if (!isset($rawScores[$target])) $rawScores[$target] = 0.0; $rawScores[$target] += $score; }
                foreach ($negativeResult['scores'] as $target => $score) { if (!isset($rawScores[$target])) $rawScores[$target] = 0.0; $rawScores[$target] += $score; }
                $themeDebug[] = array('title'=>isset($item['title']) ? (string)$item['title'] : '', 'link'=>$link, 'positive'=>$positiveResult['theme_hits'], 'negative'=>$negativeResult['theme_hits']);
                $accepted++; $newsCount++;
            }
            $feedStats[] = array('name'=>$feedName,'status'=>'ok','accepted_items'=>$accepted,'fetched_items'=>count($items));
        } catch (Throwable $e) {
            $feedStats[] = array('name'=>$feedName,'status'=>'error','reason'=>$e->getMessage());
        }
    }
    $normalizedScores = $rawScores;
    if ($newsCount > 0) { foreach ($normalizedScores as $k => $v) { $normalizedScores[$k] = $v / $newsCount; } }
    $current = array('food'=>$normalizedScores['food'] * $foodScale, 'beetle'=>$normalizedScores['beetle'] * $beetleScale, 'caterpillar'=>$normalizedScores['caterpillar'] * $caterpillarScale, 'moth'=>$normalizedScores['moth'] * $mothScale);
    $previous = readState($stateFile);
    $smoothed = array('food'=>smoothValue($previous['food'], $current['food'], $alpha), 'beetle'=>smoothValue($previous['beetle'], $current['beetle'], $alpha), 'caterpillar'=>smoothValue($previous['caterpillar'], $current['caterpillar'], $alpha), 'moth'=>smoothValue($previous['moth'], $current['moth'], $alpha));
    writeJsonFile($stateFile, $smoothed, true);
    $result = array(
        'food' => clampInt($smoothed['food'], 0, $maxFood),
        'beetle' => clampInt($smoothed['beetle'], 0, $maxBeetle),
        'caterpillar' => clampInt($smoothed['caterpillar'], 0, $maxCaterpillar),
        'moth' => clampInt($smoothed['moth'], 0, $maxMoth),
        'meta' => array(
            'generated_at_utc' => gmdate('c'),
            'news_count' => $newsCount,
            'lookback_hours' => $lookbackHours,
            'raw_scores' => $rawScores,
            'normalized_scores' => $normalizedScores,
            'smoothed_scores' => $smoothed,
            'feed_stats' => $feedStats,
            'theme_debug' => $themeDebug
        )
    );
    writeJsonFile($resultFile, $result, $prettyPrint);
    echo "OK\n";
    echo "News analyzed: " . $newsCount . "\n";
    echo "food=" . $result['food'] . ", beetle=" . $result['beetle'] . ", caterpillar=" . $result['caterpillar'] . ", moth=" . $result['moth'] . "\n";
} catch (Throwable $e) {
    fwrite(STDERR, 'ERROR: ' . $e->getMessage() . PHP_EOL);
    exit(1);
}