std/src/unicode/category.nv - main - nv-lang/nova

std/src/unicode/category.nv
301 строка11 KB
Evgeniy Golovin
lint(std/src,examples): fix W_MANUAL_SLICE_TO_END (98) — canonical open-range slices
01 авг 2026, 17:03
01 авг 2026, 17:032ece7ec
// Plan 152.3b (D253): code-point-level Unicode classification API.
//
// Opt-in `std.unicode` peer (NOT prelude — table cost paid only by importers).
// Co-equal with `category_data.nv` (generated tables) in the `std.unicode`
// folder-module, so the `*_DATA` constants are visible here directly. Reuses peer
// helpers: parse_flat / range_lookup3 / in_ranges2 (graphemes.nv); cps_to_str
// (normalize.nv); lower_one / upper_one (case.nv).
//
//   general_category(cp)  — the UCD General_Category (TR44) of a code point.
//   is_alphabetic / is_whitespace / is_numeric / is_uppercase / is_lowercase /
//   is_control / is_alphanumeric — binary predicates (1:1 with the UCD, NOT an
//   ASCII approximation).
//   char_to_uppercase(cp) / char_to_lowercase(cp) — full per-code-point case
//   mapping, returned as a str (multi-code-point: ß → "SS").
//
// These are the code-point analogues of the string-level `to_uppercase` /
// `to_lowercase` (case.nv): a single code point with no surrounding context.
// Final_Sigma is a string-level context rule, so for a lone Σ (U+03A3) the
// lowercase is σ (the non-final form) — the correct context-free answer.

module std.unicode

import std.collections.vec.{Vec}

// ─── General_Category codes (MUST match nova-codegen unicode `gc_cat_code`) ───
// Canonical UCD order (TR44 Table 12); mirrors the generator's 1..=30 mapping.
// `Cn` (code 30) is the default for any code point absent from GC_DATA.
const GC_LU = 1     // Letter, uppercase
const GC_LL = 2     // Letter, lowercase
const GC_LT = 3     // Letter, titlecase
const GC_LM = 4     // Letter, modifier
const GC_LO = 5     // Letter, other
const GC_MN = 6     // Mark, nonspacing
const GC_MC = 7     // Mark, spacing combining
const GC_ME = 8     // Mark, enclosing
const GC_ND = 9     // Number, decimal digit
const GC_NL = 10    // Number, letter
const GC_NO = 11    // Number, other
const GC_PC = 12    // Punctuation, connector
const GC_PD = 13    // Punctuation, dash
const GC_PS = 14    // Punctuation, open
const GC_PE = 15    // Punctuation, close
const GC_PI = 16    // Punctuation, initial quote
const GC_PF = 17    // Punctuation, final quote
const GC_PO = 18    // Punctuation, other
const GC_SM = 19    // Symbol, math
const GC_SC = 20    // Symbol, currency
const GC_SK = 21    // Symbol, modifier
const GC_SO = 22    // Symbol, other
const GC_ZS = 23    // Separator, space
const GC_ZL = 24    // Separator, line
const GC_ZP = 25    // Separator, paragraph
const GC_CC = 26    // Other, control
const GC_CF = 27    // Other, format
const GC_CS = 28    // Other, surrogate
const GC_CO = 29    // Other, private use
const GC_CN = 30    // Other, not assigned (default)

/// The 30 Unicode General_Category values (UCD TR44 Table 12), in canonical
/// order. `Cn` is the value of every code point not assigned a category by
/// `UnicodeData.txt` (unassigned / reserved / noncharacter).
export type GeneralCategory enum
    | Lu | Ll | Lt | Lm | Lo
    | Mn | Mc | Me
    | Nd | Nl | No
    | Pc | Pd | Ps | Pe | Pi | Pf | Po
    | Sm | Sc | Sk | So
    | Zs | Zl | Zp
    | Cc | Cf | Cs | Co | Cn

// ─── lazy-static range tables (built once on first touch) ───
// GC_DATA: (lo,hi,code) triples (stride 3) → range_lookup3. ALPHA_DATA / WS_DATA:
// (lo,hi) pairs (stride 2) → in_ranges2. All sorted by lo (binary search).
ro gc_flat []u32 = parse_flat(GC_DATA)
ro alpha_flat []u32 = parse_flat(ALPHA_DATA)
ro ws_flat []u32 = parse_flat(WS_DATA)

// Raw General_Category code (1..=30) of `cp`; 30 (`Cn`) when absent from GC_DATA
// (range_lookup3 returns 0 for "not found", which we map to the Cn default).
fn gc_code(cp u32) -> int {
    ro c = range_lookup3(gc_flat, cp)
    if c == 0 { GC_CN } else { c }
}

// Map a generator category code (1..=30) to the GeneralCategory variant.
fn gc_from_code(code int) -> GeneralCategory {
    match code {
        1  => Lu
        2  => Ll
        3  => Lt
        4  => Lm
        5  => Lo
        6  => Mn
        7  => Mc
        8  => Me
        9  => Nd
        10 => Nl
        11 => No
        12 => Pc
        13 => Pd
        14 => Ps
        15 => Pe
        16 => Pi
        17 => Pf
        18 => Po
        19 => Sm
        20 => Sc
        21 => Sk
        22 => So
        23 => Zs
        24 => Zl
        25 => Zp
        26 => Cc
        27 => Cf
        28 => Cs
        29 => Co
        _  => Cn
    }
}

// ─── public API ───

/// The UCD General_Category of code point `cp`. Returns `Cn` (not assigned) for
/// any code point absent from `UnicodeData.txt` (unassigned/reserved/noncharacter)
/// and for out-of-range values.
export fn general_category(cp u32) -> GeneralCategory => gc_from_code(gc_code(cp))

/// Alphabetic (UCD `Alphabetic` derived core property). Broader than `Lu|Ll|Lt`:
/// includes `Lm|Lo|Nl` and Other_Alphabetic marks, exactly per DerivedCoreProperties.
export fn is_alphabetic(cp u32) -> bool => in_ranges2(alpha_flat, cp)

/// White_Space (UCD `White_Space` binary property from PropList.txt).
export fn is_whitespace(cp u32) -> bool => in_ranges2(ws_flat, cp)

/// Numeric: General_Category ∈ {Nd, Nl, No} (matches Rust `char::is_numeric` and
/// the generator note — derived from GC, no separate table).
export fn is_numeric(cp u32) -> bool {
    ro c = gc_code(cp)
    c == GC_ND || c == GC_NL || c == GC_NO
}

/// Uppercase letter: General_Category == Lu.
export fn is_uppercase(cp u32) -> bool => gc_code(cp) == GC_LU

/// Lowercase letter: General_Category == Ll.
export fn is_lowercase(cp u32) -> bool => gc_code(cp) == GC_LL

/// Control character: General_Category == Cc.
export fn is_control(cp u32) -> bool => gc_code(cp) == GC_CC

/// Alphanumeric: alphabetic OR numeric.
export fn is_alphanumeric(cp u32) -> bool => is_alphabetic(cp) || is_numeric(cp)

/// Full uppercase mapping of a single code point, as a str. Multi-code-point
/// expansions apply (ß → "SS", ﬁ → "FI"); no locale tailoring. Reuses the
/// case.nv `UPPER_DATA` table via `upper_one`.
export fn char_to_uppercase(cp u32) -> str => cps_to_str(upper_one(cp))

/// Full lowercase mapping of a single code point, as a str. Final_Sigma is a
/// string-level context rule, so a lone Σ (U+03A3) lowercases to σ (U+03C3, the
/// non-final form) — the correct context-free answer. Reuses `LOWER_DATA` via
/// `lower_one`.
export fn char_to_lowercase(cp u32) -> str => cps_to_str(lower_one(cp))

// ─── char Unicode-aware methods (Plan 169.2.1 — moved back here) ───
// Plan 162 Ф.4 hoisted these to std/prelude/core.nv (so they were available
// without `import std.unicode`), but that forced `core.nv` to `import
// std.unicode`, which pulled the whole unicode folder-module — including
// `normalize.nv::cps_to_str`'s `consume sb = StringBuilder…` — into every
// partial `#prelude(core, …)`. D133 (type-check, before DCE) then saw `sb`
// not-consumed because StringBuilder's consume-methods live in `collections`,
// which the partial prelude does not pull → plan107 failed. Plan 169.2.1 (D303)
// moves the methods back into `std.unicode` and makes them prelude-available
// via a re-export in `std/prelude.nv` (the rolling facade) instead of via
// `core`. `core` becomes unicode-free again.
//
// The bodies delegate to the code-point predicates / case-mapping helpers
// declared above in this same `std.unicode` folder-module — no import needed.

/// Unicode Alphabetic property (UCD `Alphabetic` derived core property).
/// Broader than `Lu|Ll|Lt`: includes `Lm|Lo|Nl` and Other_Alphabetic marks.
/// Equivalent to `std.unicode.is_alphabetic(@ as u32)`.
export fn char @is_alphabetic() -> bool => is_alphabetic(@ as u32)

/// Numeric: General_Category ∈ {Nd, Nl, No}.
/// Matches Rust `char::is_numeric` — derived from UCD GC, not ASCII-only.
export fn char @is_numeric() -> bool => is_numeric(@ as u32)

/// Alphanumeric: alphabetic OR numeric.
export fn char @is_alphanumeric() -> bool => is_alphanumeric(@ as u32)

/// White_Space (UCD `White_Space` binary property from PropList.txt).
export fn char @is_whitespace() -> bool => is_whitespace(@ as u32)

/// Uppercase letter: General_Category == Lu.
export fn char @is_uppercase() -> bool => is_uppercase(@ as u32)

/// Lowercase letter: General_Category == Ll.
export fn char @is_lowercase() -> bool => is_lowercase(@ as u32)

/// Control character: General_Category == Cc.
export fn char @is_control() -> bool => is_control(@ as u32)

/// UCD General_Category of this code point. Returns `Cn` (not assigned)
/// for unassigned/reserved/noncharacter code points.
export fn char @general_category() -> GeneralCategory => general_category(@ as u32)

/// Full uppercase mapping of a single code point, as a str.
/// Multi-code-point expansions apply (ß → "SS", ﬁ → "FI"); no locale tailoring.
export fn char @to_uppercase() -> str => char_to_uppercase(@ as u32)

/// Full lowercase mapping of a single code point, as a str.
/// Final_Sigma is a string-level context rule; a lone Σ lowercases to σ.
export fn char @to_lowercase() -> str => char_to_lowercase(@ as u32)

// ─── Unicode trim / split (Plan 91.18 Ф.5) ───
// Opt-in under `import std.unicode`; bare names (no _ascii_ suffix) shadow
// nothing — ASCII variants are named trim_ascii / trim_ascii_start / trim_ascii_end.

// Trim Unicode whitespace from both ends (zero-copy view).
// Uses is_whitespace (WS_DATA) and decode_at (cp_utils.nv peer).
fn str @trim_unicode() -> str {
    ro n = @byte_len()
    if n == 0 { return @ }
    ro bytes = @bytes()
    mut start = 0
    while start < n {
        ro (cp, step) = decode_at(@, start)
        if !is_whitespace(cp) { break }
        start += step
    }
    mut end = n
    while end > start {
        mut i = end - 1
        while i > start && ((bytes[i] as int) & 0xC0) == 0x80 { i -= 1 }
        ro (cp, _) = decode_at(@, i)
        if !is_whitespace(cp) { break }
        end = i
    }
    @[start..end]
}

// Trim Unicode whitespace from the start (zero-copy view).
fn str @trim_unicode_start() -> str {
    ro n = @byte_len()
    mut start = 0
    while start < n {
        ro (cp, step) = decode_at(@, start)
        if !is_whitespace(cp) { break }
        start += step
    }
    @[start..n]
}

// Trim Unicode whitespace from the end (zero-copy view).
fn str @trim_unicode_end() -> str {
    ro n = @byte_len()
    ro bytes = @bytes()
    mut end = n
    while end > 0 {
        mut i = end - 1
        while i > 0 && ((bytes[i] as int) & 0xC0) == 0x80 { i -= 1 }
        ro (cp, _) = decode_at(@, i)
        if !is_whitespace(cp) { break }
        end = i
    }
    @[..end]
}

/// Trim Unicode whitespace from both ends (opt-in under `import std.unicode`).
export fn str @trim() -> str => @trim_unicode()

/// Trim Unicode whitespace from the start (opt-in under `import std.unicode`).
export fn str @trim_start() -> str => @trim_unicode_start()

/// Trim Unicode whitespace from the end (opt-in under `import std.unicode`).
export fn str @trim_end() -> str => @trim_unicode_end()

/// Split on runs of Unicode whitespace, skipping empty segments.
/// Opt-in under `import std.unicode`.
export fn str @split_whitespace() -> ro []str {
    mut out = []str.new()
    ro n = @byte_len()
    mut i = 0
    while i < n {
        while i < n {
            ro (cp, step) = decode_at(@, i)
            if !is_whitespace(cp) { break }
            i += step
        }
        if i >= n { break }
        ro start = i
        while i < n {
            ro (cp, step) = decode_at(@, i)
            if is_whitespace(cp) { break }
            i += step
        }
        out.push(@[start..i])
    }
    out
}