std/src/unicode/cp_utils.nv - main - nv-lang/nova

std/src/unicode/cp_utils.nv
139 строк5 KB
Evgeniy Golovin
std(naming): collections/hashmap → hash_map — единственный нарушитель snake_case-правила модулей
06 авг 2026, 12:48
06 авг 2026, 12:480b0a260
// Codepoint utilities shared across std.unicode peers: hex parser, table parsers,
// codepoint <-> str converters, UTF-8 decode.
// Dedup helpers: parse_cp_keyed_table / parse_cp_int_table consolidate the 6
// identical semicolon-of-colon table parsers scattered across normalize.nv and
// collate.nv into 2 shared functions.

module std.unicode

import std.collections.hash_map.{HashMap}
import std.collections.vec.{Vec}

// Parse a lowercase-hex codepoint/weight as `u32` (D54/D77: `None` on malformed
// input instead of a `-1` int sentinel — codepoints/weights are `u32`, and a
// fallible parse returns `Option`). The generated tables are always well-formed,
// so `None` is a safety net, not a hot path.
fn hex(s str) -> Option[u32] {
    // Plan 174.1: `str.parse_int` retracted → `s.to_int(radix: 16)`.
    match s.to_int(radix: 16) { Ok(v) => Some(v as u32), Err(_) => None }
}

// "d1,d2,d3" (comma-separated hex) -> [d1, d2, d3].
fn parse_cp_list(s str) -> []u32 {
    mut out []u32 = []u32.new(cap: 4)
    for part in s.split(",") {
        match hex(part) { Some(cp) => out.push(cp), None => {} }
    }
    out
}

// "cp:value;cp2:value2;.." -> { cp -> value_str }.
// Covers decomposition tables (NFD/NFKD), collation single/contraction tables —
// all share this exact semicolon-of-colon text shape.
fn parse_cp_keyed_table(data str) -> HashMap[u32, str] {
    ro entries = data.split(";").collect()
    mut m = HashMap[u32, str].new(cap: entries.len() + 8)
    for entry in entries {
        // codepoint key is `u32` (D327); `hex` yields `Option[u32]` (D54/D77).
        match entry.split_once(":") {
            Some((k, v)) => {
                match hex(k) { Some(cp) => m.insert(cp, v), None => {} }
            }
            None => {}
        }
    }
    m
}

// "cp:int;cp2:int2;.." -> { cp -> int }.
// Used for CCC table (canonical combining class).
fn parse_cp_int_table(data str) -> HashMap[u32, int] {
    ro entries = data.split(";").collect()
    mut m = HashMap[u32, int].new(cap: entries.len() + 8)
    for entry in entries {
        // codepoint key `u32` (D327); CCC value `int` (a small class number, NOT a
        // codepoint — it is a measure/class, so `int` per D226).
        match entry.split_once(":") {
            Some((k, v)) => {
                match hex(k) {
                    Some(cp) => match hex(v) { Some(cc) => m.insert(cp, cc as int), None => {} }
                    None => {}
                }
            }
            None => {}
        }
    }
    m
}

// "a,b:cp;..." -> { (a << 21) | b -> cp }. Packed-int key avoids tuple-key
// HashMap (codepoints fit in 21 bits; see [M-hashmap-tuple-key-mono]).
fn parse_comp_table(data str) -> HashMap[int, u32] {
    ro entries = data.split(";").collect()
    mut m = HashMap[int, u32].new(cap: entries.len() + 8)
    for entry in entries {
        // Key is a packed pair `(a<<21)|b` (> 32 bits → `int`, D327 §bit-packing);
        // value is the composed codepoint → `u32`. `compose_pair` returns
        // `Option[u32]` (no `-1` sentinel — D327 §fallible).
        match entry.split_once(":") {
            Some((ab, cp_s)) => {
                match ab.split_once(",") {
                    Some((a_s, b_s)) => {
                        match hex(a_s) {
                            Some(a) => match hex(b_s) {
                                Some(b) => match hex(cp_s) {
                                    Some(cp) => m.insert(((a as int) << 21) | (b as int), cp)
                                    None => {}
                                }
                                None => {}
                            }
                            None => {}
                        }
                    }
                    None => {}
                }
            }
            None => {}
        }
    }
    m
}

fn str_to_cps(s str) -> []u32 {
    mut out []u32 = []u32.new(cap: s.byte_len())
    for c in s.chars() { out.push(c as u32) }
    out
}

fn cps_to_str(cps []u32) -> str {
    consume sb = StringBuilder.new(cap: cps.len() * 2)
    for cp in cps {
        ro c = (cp as int).to_char() ?? '\u{FFFD}'
        sb.append(c)
    }
    sb
}

// UTF-8 decode at byte offset `i`: returns (codepoint, byte_step). R-UTF8 (D26)
// makes this total; the `else` arm guards a truncated tail / invalid lead → U+FFFD, step 1.
// Intentional per-module copy of UTF-8 decode; runtime.string has its own (folder boundary).
// Codepoint is returned as `u32` (D327 §flow — codepoints flow as `u32`); the decode
// itself is byte/bit work in `int`, cast to `u32` at the boundary. `step` is a byte
// count, so `int` (D226 — a measure, not a codepoint).
fn decode_at(s str, i int) -> (u32, int) {
    ro bytes = s.bytes()
    ro n = s.byte_len()
    ro b = bytes[i] as int
    if b < 0x80 {
        (b as u32, 1)
    } else if ((b & 0xE0) == 0xC0) && (i + 1 < n) {
        ((((b & 0x1F) << 6) | (bytes[i+1] as int & 0x3F)) as u32, 2)
    } else if ((b & 0xF0) == 0xE0) && (i + 2 < n) {
        ((((b & 0x0F) << 12) | ((bytes[i+1] as int & 0x3F) << 6) | (bytes[i+2] as int & 0x3F)) as u32, 3)
    } else if ((b & 0xF8) == 0xF0) && (i + 3 < n) {
        ((((b & 0x07) << 18) | ((bytes[i+1] as int & 0x3F) << 12) | ((bytes[i+2] as int & 0x3F) << 6) | (bytes[i+3] as int & 0x3F)) as u32, 4)
    } else {
        // U+FFFD: повреждённая/оборванная последовательность видима (канон Unicode).
        (0xFFFD as u32, 1)
    }
}