onec/tui/src/parser.rs - master - nkolentcev/devscripts

onec/tui/src/parser.rs
492 строки15 KB
Nikolay Kolentcev
orc: стриминг загрузок, фиксы парсера и учётки; README со скриншотом
01 май 2026, 16:54
01 май 2026, 16:54f6ab2e0
//! HTML-парсер страниц releases.1c.ru (логика близка к v8platform/oneget).

use std::collections::HashSet;
use std::io::Read;
use std::sync::OnceLock;

use chrono::{NaiveDate, NaiveTime, TimeZone, Utc};
use regex::Regex;
use scraper::{Html, Selector};
use thiserror::Error;

#[derive(Debug, Error)]
pub enum ParseError {
    #[error("ответ не UTF-8")]
    InvalidUtf8,
    #[error("в ответе нет списка проектов (учётка ИТС, доступ к releases; см. orc --dump-total)")]
    NoProjectsOnPage,
    #[error("таблица версий не найдена")]
    VersionsTable,
}

#[derive(Debug, Clone)]
pub struct ProjectRow {
    pub id: String,
    pub group: String,
    pub name: String,
}

#[derive(Debug, Clone)]
pub struct VersionRow {
    pub url: String,
    pub name: String,
    pub published: Option<chrono::DateTime<chrono::Utc>>,
}

#[derive(Debug, Clone)]
pub struct ReleaseFile {
    pub url: String,
}

static RELEASES_HOST: &str = "https://releases.1c.ru";

/// Ответ 200, но вместо каталога/проекта отдана форма входа `login.1c.ru` (без редиректа 401).
pub fn is_login_page_html(html: &str) -> bool {
    html.to_lowercase().contains("<title>личные данные</title>")
}

pub fn parse_total_catalog(html: &str) -> Result<Vec<ProjectRow>, ParseError> {
    let strict = parse_total_strict(html);
    if !strict.is_empty() {
        return Ok(strict);
    }

    let loose = parse_total_from_project_links(html);
    if !loose.is_empty() {
        return Ok(loose);
    }

    let rex = parse_total_from_regex(html);
    if !rex.is_empty() {
        return Ok(rex);
    }

    Err(ParseError::NoProjectsOnPage)
}

pub fn parse_total_catalog_bytes(body: &[u8]) -> Result<Vec<ProjectRow>, ParseError> {
    let html = std::str::from_utf8(body).map_err(|_| ParseError::InvalidUtf8)?;
    parse_total_catalog(html)
}

fn sel(s: &str) -> &'static Selector {
    use std::collections::HashMap;
    use std::sync::OnceLock;
    use std::sync::Mutex;
    static CACHE: OnceLock<Mutex<HashMap<&'static str, &'static Selector>>> = OnceLock::new();
    let map = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
    let mut g = map.lock().expect("selector cache poisoned");
    if let Some(&s_ref) = g.get(s) {
        return s_ref;
    }
    let key: &'static str = Box::leak(s.to_string().into_boxed_str());
    let parsed: &'static Selector = Box::leak(Box::new(
        Selector::parse(s).unwrap_or_else(|_| panic!("invalid selector: {s}")),
    ));
    g.insert(key, parsed);
    parsed
}

fn parse_total_strict(html: &str) -> Vec<ProjectRow> {
    let doc = Html::parse_document(html);
    let Some(table) = doc.select(sel("[id$='actualTable']")).next() else {
        return Vec::new();
    };

    let mut out = Vec::new();

    for group in table.select(sel("[group]")) {
        let group_id = group.value().attr("group").unwrap_or("").to_string();
        if group_id.is_empty() {
            continue;
        }
        let group_name = group
            .select(sel(".group-name"))
            .next()
            .map(|n| trim_text(&n.text().collect::<String>()))
            .unwrap_or_default();

        // Сравниваем `parent-group` в Rust, а не подставляем в селектор:
        // group_id из HTML мог бы содержать кавычки/бэкслэши и сломать парсер CSS.
        for row in table.select(sel("[parent-group]")) {
            let pg = row.value().attr("parent-group").unwrap_or("");
            if !pg.ends_with(group_id.as_str()) {
                continue;
            }

            let Some(name_td) = row.select(sel(".nameColumn")).next() else {
                continue;
            };

            let name = trim_text(&name_td.text().collect::<String>());
            let href = name_td
                .select(sel("a"))
                .next()
                .and_then(|a| a.value().attr("href"))
                .unwrap_or("")
                .to_string();
            let Some(id) = project_id_from_href(&href) else {
                continue;
            };

            out.push(ProjectRow {
                id,
                group: group_name.clone(),
                name,
            });
        }
    }

    out
}

fn project_id_from_href(href: &str) -> Option<String> {
    let h = href.trim();
    let tail = if let Some(rest) = h.strip_prefix("/project/") {
        rest
    } else if let Some(idx) = h.find("/project/") {
        &h[idx + "/project/".len()..]
    } else {
        return None;
    };
    let id = tail
        .split(|c| c == '?' || c == '#')
        .next()
        .unwrap_or("")
        .trim_end_matches('/');
    if id.is_empty() {
        return None;
    }
    Some(id.to_string())
}

fn parse_total_from_project_links(html: &str) -> Vec<ProjectRow> {
    let doc = Html::parse_document(html);
    let mut seen = HashSet::new();
    let mut out = Vec::new();

    let selector_strings = [
        r#"a[href*="/project/"]"#,
        r#"[to*="/project/"]"#,
        r#"a[data-href*="/project/"]"#,
        r#"[data-url*="/project/"]"#,
    ];

    for sel_str in selector_strings {
        let Ok(sel) = Selector::parse(sel_str) else {
            continue;
        };
        for el in doc.select(&sel) {
            let href = el
                .value()
                .attr("href")
                .or_else(|| el.value().attr("to"))
                .or_else(|| el.value().attr("data-href"))
                .or_else(|| el.value().attr("data-url"));
            let Some(href) = href else {
                continue;
            };
            let Some(id) = project_id_from_href(href) else {
                continue;
            };
            if !seen.insert(id.clone()) {
                continue;
            }
            let name = trim_text(&el.text().collect::<String>());
            let name = if name.is_empty() {
                id.clone()
            } else {
                name
            };
            out.push(ProjectRow {
                id,
                group: String::new(),
                name,
            });
        }
    }

    out.sort_by(|a, b| a.name.cmp(&b.name));
    out
}

/// Ссылки в JSON внутри `<script>` или экранированные в атрибутах.
pub(crate) fn parse_total_from_regex(html: &str) -> Vec<ProjectRow> {
    static RE: OnceLock<Regex> = OnceLock::new();
    let re = RE.get_or_init(|| {
        Regex::new(r#"(?i)/project/([-A-Za-z0-9_.]+)"#).expect("regex")
    });

    let mut seen = HashSet::new();
    let mut out = Vec::new();

    for cap in re.captures_iter(html) {
        let id = cap[1].to_string();
        if !seen.insert(id.clone()) {
            continue;
        }
        out.push(ProjectRow {
            id: id.clone(),
            group: String::new(),
            name: id,
        });
    }

    out.sort_by(|a, b| a.name.cmp(&b.name));
    out
}

pub fn parse_project_versions<R: Read>(body: R) -> Result<Vec<VersionRow>, ParseError> {
    let mut html = String::new();
    let mut r = body;
    r.read_to_string(&mut html).map_err(|_| ParseError::InvalidUtf8)?;
    let doc = Html::parse_document(&html);
    let table_sel = Selector::parse("[id$='versionsTable']").expect("selector");
    let table = doc
        .select(&table_sel)
        .next()
        .ok_or(ParseError::VersionsTable)?;

    let tr_sel = Selector::parse("tr").unwrap();
    let mut rows = Vec::new();

    for tr in table.select(&tr_sel) {
        if tr
            .parent()
            .and_then(|p| p.value().as_element())
            .map(|e| e.name() == "thead")
            .unwrap_or(false)
        {
            continue;
        }

        let ver_a = tr
            .select(&Selector::parse(".versionColumn a").unwrap())
            .next();
        let Some(ver_a) = ver_a else { continue };

        let url = ver_a.value().attr("href").unwrap_or("").to_string();
        let name = trim_text(&ver_a.text().collect::<String>());
        let date_text = trim_text(
            &tr.select(&Selector::parse(".dateColumn").unwrap())
                .next()
                .map(|d| d.text().collect::<String>())
                .unwrap_or_default(),
        );
        let published = parse_date(&date_text);
        rows.push(VersionRow {
            url,
            name,
            published,
        });
    }

    Ok(rows)
}

pub fn parse_release_files_list<R: Read>(body: R) -> Result<Vec<ReleaseFile>, ParseError> {
    let mut html = String::new();
    let mut r = body;
    r.read_to_string(&mut html).map_err(|_| ParseError::InvalidUtf8)?;
    let doc = Html::parse_document(&html);
    let sel = Selector::parse(".files-container .formLine a").unwrap();

    let mut rows = Vec::new();
    for a in doc.select(&sel) {
        let url = a.value().attr("href").unwrap_or("").to_string();
        if !url.starts_with("/version_file") {
            continue;
        }
        rows.push(ReleaseFile { url });
    }
    Ok(rows)
}

pub fn parse_download_dist_links<R: Read>(body: R) -> Result<Vec<String>, ParseError> {
    let mut html = String::new();
    let mut r = body;
    r.read_to_string(&mut html).map_err(|_| ParseError::InvalidUtf8)?;
    let doc = Html::parse_document(&html);
    let sel = Selector::parse(".downloadDist a").unwrap();
    let mut hrefs = Vec::new();
    for a in doc.select(&sel) {
        if let Some(h) = a.value().attr("href") {
            hrefs.push(h.to_string());
        }
    }
    Ok(hrefs)
}

fn trim_text(s: &str) -> String {
    s.split_whitespace().collect::<Vec<_>>().join(" ")
}

fn parse_date(s: &str) -> Option<chrono::DateTime<chrono::Utc>> {
    let s = s.trim();
    if s.is_empty() {
        return None;
    }
    let mid = NaiveTime::from_hms_opt(0, 0, 0)?;
    // 4-значный год — однозначный; %y оставлен фоллбэком (chrono трактует 50..99 как 19xx).
    for fmt in ["%d.%m.%Y", "%d.%m.%y"] {
        if let Ok(d) = NaiveDate::parse_from_str(s, fmt) {
            return Some(Utc.from_utc_datetime(&d.and_time(mid)));
        }
    }
    None
}

pub fn is_platform_row(row: &ProjectRow) -> bool {
    let g = row.group.to_lowercase();
    let n = row.name.to_lowercase();
    let id = row.id.to_lowercase();
    g.contains("платформ")
        || n.contains("платформ")
        || id.contains("platform")
        || id == "platform83"
        || id == "platform85"
}

pub fn platform_line(ver: &str) -> &'static str {
    let v = ver.trim();
    if v.starts_with("8.5") {
        "8.5"
    } else if v.starts_with("8.3") {
        "8.3"
    } else {
        "прочее"
    }
}

pub fn absolute_url(path: &str) -> String {
    if path.starts_with("http") {
        path.to_string()
    } else {
        format!("{}{}", RELEASES_HOST, path)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn project_id_from_href_relative_and_absolute() {
        assert_eq!(
            project_id_from_href("/project/platform83"),
            Some("platform83".into())
        );
        assert_eq!(
            project_id_from_href("https://releases.1c.ru/project/foo_bar?x=1"),
            Some("foo_bar".into())
        );
        assert_eq!(project_id_from_href("/foo"), None);
    }

    #[test]
    fn is_platform_row_heuristic() {
        let p = |id: &str, group: &str, name: &str| {
            is_platform_row(&ProjectRow {
                id: id.into(),
                group: group.into(),
                name: name.into(),
            })
        };
        assert!(p("platform83", "", ""));
        assert!(p("platform85", "", ""));
        assert!(p("x", "Что-то платформы", ""));
        assert!(p("x", "", "Платформа предприятия"));
        assert!(p("my_platform_x", "", ""));
        assert!(!p("accounting", "", "Бухгалтерия"));
    }

    #[test]
    fn platform_line_order_85_before_83() {
        // 8.5 должен определяться раньше общего префикса 8.3
        assert_eq!(platform_line("8.5.1"), "8.5");
        assert_eq!(platform_line("  8.3.27 "), "8.3");
        assert_eq!(platform_line("8.4.1"), "прочее");
    }

    #[test]
    fn parse_total_strict_minimal_table() {
        let html = r##"
<table id="wrap_actualTable">
<tr group="gid">
  <td><span class="group-name">Группа</span></td>
</tr>
<tr parent-group="pre_gid">
  <td class="nameColumn"><a href="/project/testproj">Имя проекта</a></td>
</tr>
</table>
"##;
        let rows = parse_total_strict(html);
        assert_eq!(rows.len(), 1);
        assert_eq!(rows[0].id, "testproj");
        assert_eq!(rows[0].name, "Имя проекта");
        assert_eq!(rows[0].group, "Группа");
    }

    #[test]
    fn parse_total_loose_links() {
        let html = r#"<html><body><a href="/project/from_a">A</a><span to="/project/from_to">T</span></body></html>"#;
        let rows = parse_total_from_project_links(html);
        assert!(rows.iter().any(|r| r.id == "from_a"));
        assert!(rows.iter().any(|r| r.id == "from_to"));
    }

    #[test]
    fn parse_total_regex_ids() {
        let html = "blah /project/Conf_Accounting32/ more";
        let rows = parse_total_from_regex(html);
        assert_eq!(rows.len(), 1);
        assert_eq!(rows[0].id, "Conf_Accounting32");
    }

    #[test]
    fn parse_project_versions_table() {
        let html = r##"
<table id="p_versionsTable">
<thead><tr><th>v</th><th>d</th></tr></thead>
<tbody>
<tr>
  <td class="versionColumn"><a href="/version_x">8.3.27.1234</a></td>
  <td class="dateColumn">15.03.24</td>
</tr>
</tbody>
</table>
"##;
        let rows = parse_project_versions(html.as_bytes()).unwrap();
        assert_eq!(rows.len(), 1);
        assert_eq!(rows[0].name, "8.3.27.1234");
        assert!(rows[0].url.contains("version_x"));
    }

    #[test]
    fn is_login_page_detects_personal_data_title() {
        assert!(is_login_page_html(
            "<!DOCTYPE html><html><head><title>Личные данные</title>"
        ));
        assert!(!is_login_page_html(
            "<title>Релизы</title><table id=\"x_actualTable\">"
        ));
    }

    /// Фильтр линий в UI (клавиши 2/3) дублирует префиксы имени версии.
    #[test]
    fn line_filter_matches_version_name_prefix() {
        let v83 = VersionRow {
            url: "/v".into(),
            name: "8.3.1".into(),
            published: None,
        };
        let v85 = VersionRow {
            url: "/v".into(),
            name: "8.5.1".into(),
            published: None,
        };
        assert!(v83.name.trim().starts_with("8.3"));
        assert!(v85.name.trim().starts_with("8.5"));
        assert!(!v85.name.trim().starts_with("8.3"));
    }
}