src/unicode_scripts.c - master - auroraos/mirror_libpostal

src/unicode_scripts.c

81 строка2 KB

[fix/utf8] reviewed and fixed all points where utf8proc_iterate is called and may return an error which can cause the iteration not to make forward progress. This includes fixing a bug where injecting invalid UTF-8 through a series of HTML-encoded codepoints can cause the C library to hang. Note: we're not fixing all the garbage encoding in the world, so if encoding is bad the output of expand_address may not be useful but it won't hang. Fixes #448

02 июл 2025, 07:10

02 июл 2025, 07:1095e97c0

#include "unicode_scripts.h"

#include "unicode_scripts_data.c"

#define MAX_ASCII 128

inline script_t get_char_script(uint32_t ch) {
    if (ch > NUM_CODEPOINTS - 1) return SCRIPT_UNKNOWN;
    return char_scripts[ch];
}

inline script_languages_t get_script_languages(script_t script) {
    return script_languages[script];
}

static inline bool is_common_script(script_t script) {
    return script == SCRIPT_COMMON || script == SCRIPT_INHERITED;
}

string_script_t get_string_script(char *str, size_t len) {
    int32_t ch;
    script_t last_script = SCRIPT_UNKNOWN;
    script_t script = SCRIPT_UNKNOWN;

    uint8_t *ptr = (uint8_t *)str;

    size_t script_len = 0;
    size_t idx = 0;

    bool is_ascii = true;

    while (idx < len) {
        ssize_t char_len = utf8proc_iterate(ptr, len, &ch);

        if (char_len <= 0 ||ch == 0) break;

        script = get_char_script((uint32_t)ch);

        if (is_common_script(script) && last_script != SCRIPT_UNKNOWN) {
            script = last_script;
        }

        if (last_script != script && last_script != SCRIPT_UNKNOWN && !is_common_script(last_script)) {
            if (script_len < len) {
                while (true) {
                    char_len = utf8proc_iterate_reversed((const uint8_t *)str, idx, &ch);
                    if (ch == 0) break;

                    /* Note: don't need to check char_len < 0 here because we're rewinding
                    ** previously valid UTF-8 characters and if anything invalid is detected,
                    ** we break out of the outer loop.
                    **/

                    script = get_char_script((uint32_t)ch);
                    if (!is_common_script(script)) {
                        break;
                    }

                    script_len -= char_len;
                    ptr -= char_len;
                    idx -= char_len;
                }
            }

            break;
        }

        is_ascii = is_ascii && ch < MAX_ASCII;

        ptr += char_len;
        idx += char_len;
        script_len += char_len;

        if (script != SCRIPT_UNKNOWN) {
            last_script = script;
        }
    
    }

    return (string_script_t) {last_script, script_len, is_ascii};
}