vshmidt/llama.cpp: LLM inference in C/C++

tests/peg-parser/simple-tokenize.cpp

37 строк863 B

Aldehir Rojas

common : introduce composable PEG parser combinators for chat parsing (#17136)

03 дек 2025, 13:45

03 дек 2025, 13:450a8026e

#include "simple-tokenize.h"

std::vector<std::string> simple_tokenize(const std::string & input) {
    std::vector<std::string> result;
    std::string              current;

    for (size_t i = 0; i < input.size(); i++) {
        switch (input[i]) {
            case ' ':
            case '\n':
            case '\t':
            case '{':
            case '}':
            case ',':
            case '[':
            case '"':
            case ']':
            case '.':
            case '<':
            case '>':
            case '=':
            case '/':
                if (!current.empty()) {
                    result.push_back(current);
                    current.clear();
                }
            default:;
        }
        current += input[i];
    }

    if (!current.empty()) {
        result.push_back(current);
    }

    return result;
}