src/lexer.cpp - master - vit1251/cmm

src/lexer.cpp
314 строк11 KB
Vitold S
Commits added a large set of example and demo programs,
26 июл 2026, 14:31
26 июл 2026, 14:318e0776e
#include "lexer.h"
#include <cctype>
#include <cstdlib>

const std::unordered_map<std::string, TokenType> Lexer::keywords_ = {
    {"fn", TokenType::FN},
    {"use", TokenType::USE},
    {"if", TokenType::IF},
    {"else", TokenType::ELSE},
    {"for", TokenType::FOR},
    {"let", TokenType::LET},
    {"const", TokenType::CONST},
    {"true", TokenType::TRUE_},
    {"false", TokenType::FALSE_},
    {"struct", TokenType::STRUCT},
    {"impl", TokenType::IMPL},
    {"extern", TokenType::EXTERN},
    {"enum", TokenType::ENUM},
    {"self", TokenType::SELF},
    {"drop", TokenType::DROP},
    {"pub", TokenType::PUB},
    {"in", TokenType::IN},
    {"as", TokenType::AS},
    {"defer", TokenType::DEFER},
    {"return", TokenType::RETURN},
    {"i8", TokenType::TYPE_I8},
    {"i16", TokenType::TYPE_I16},
    {"i32", TokenType::TYPE_I32},
    {"i64", TokenType::TYPE_I64},
    {"u8", TokenType::TYPE_U8},
    {"u16", TokenType::TYPE_U16},
    {"u32", TokenType::TYPE_U32},
    {"u64", TokenType::TYPE_U64},
    {"f32", TokenType::TYPE_F32},
    {"f64", TokenType::TYPE_F64},
    {"bool", TokenType::TYPE_BOOL},
    {"void", TokenType::TYPE_VOID},
};

Lexer::Lexer(std::string source, std::string filename)
    : source_(std::move(source)), filename_(std::move(filename)) {
    advance();
}

char Lexer::peek_char(size_t ahead) const {
    size_t p = pos_ + ahead;
    return p < source_.size() ? source_[p] : '\0';
}

char Lexer::next_char() {
    if (pos_ >= source_.size()) return '\0';
    char c = source_[pos_++];
    if (c == '\n') { line_++; col_ = 1; }
    else { col_++; }
    return c;
}

void Lexer::skip_whitespace_and_comments() {
    while (true) {
        char c = peek_char();
        if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
            next_char();
            continue;
        }
        if (c == '/' && peek_char(1) == '/') {
            while (peek_char() && peek_char() != '\n') next_char();
            continue;
        }
        if (c == '/' && peek_char(1) == '*') {
            next_char(); next_char();
            while (peek_char() && !(peek_char() == '*' && peek_char(1) == '/')) next_char();
            if (peek_char()) { next_char(); next_char(); }
            continue;
        }
        break;
    }
}

Token Lexer::read_number(char first) {
    size_t sl = line_, sc = col_ - 1;
    std::string text(1, first);
    bool is_float = false;

    // Hex literal: 0x...
    if (first == '0' && (peek_char() == 'x' || peek_char() == 'X')) {
        text += next_char(); // x
        while (std::isxdigit(peek_char())) text += next_char();
        Token t;
        t.type = TokenType::NUMBER;
        t.lexeme = text;
        t.line = sl; t.col = sc;
        t.int_value = std::stoll(text, nullptr, 0);
        t.float_value = 0.0;
        // Check for literal type suffix after hex
        if (std::isalpha(peek_char())) {
            std::string suffix;
            suffix += peek_char(0);
            if (std::isalnum(peek_char(1))) suffix += peek_char(1);
            if (std::isalnum(peek_char(2))) suffix += peek_char(2);
            if (suffix == "u8" || suffix == "i8" || suffix == "u16" || suffix == "i16" ||
                suffix == "u32" || suffix == "i32" || suffix == "u64" || suffix == "i64") {
                next_char();
                if (std::isalnum(peek_char())) next_char();
                if (std::isalnum(peek_char())) next_char();
                t.literal_type = suffix;
            }
        }
        return t;
    }

    while (std::isdigit(peek_char())) text += next_char();
    if (peek_char() == '.' && peek_char(1) != '.' && std::isdigit(peek_char(1)) && !is_float) {
        is_float = true;
        text += next_char();
        while (std::isdigit(peek_char())) text += next_char();
    }
    Token t;
    t.type = is_float ? TokenType::FLOAT : TokenType::NUMBER;
    t.lexeme = text;
    t.line = sl; t.col = sc;

    // Check for literal type suffix (u8, u16, u32, u64, i8, i16, i32, i64, f32, f64)
    if (!is_float && std::isalpha(peek_char())) {
        std::string suffix;
        suffix += peek_char(0);
        if (std::isalnum(peek_char(1))) suffix += peek_char(1);
        if (std::isalnum(peek_char(2))) suffix += peek_char(2);

        if (suffix == "u8" || suffix == "i8" ||
            suffix == "u16" || suffix == "i16" ||
            suffix == "u32" || suffix == "i32" ||
            suffix == "u64" || suffix == "i64" ||
            suffix == "f32" || suffix == "f64") {
            // Consume suffix characters
            next_char();
            if (std::isalnum(peek_char())) next_char();
            if (std::isalnum(peek_char())) next_char();
            t.literal_type = suffix;
            if (suffix[0] == 'f') {
                is_float = true;
                t.type = TokenType::FLOAT;
                t.float_value = std::stod(text);
                t.int_value = 0;
            } else {
                t.int_value = std::stoll(text);
                t.float_value = 0.0;
            }
            return t;
        }
    }

    if (is_float) {
        t.float_value = std::stod(text);
        t.int_value = 0;
    } else {
        t.int_value = std::stoll(text);
        t.float_value = 0.0;
    }
    return t;
}

Token Lexer::read_string() {
    size_t sl = line_, sc = col_ - 1;
    std::string text;
    while (peek_char() && peek_char() != '"') {
        if (peek_char() == '\\') {
            next_char();
            switch (peek_char()) {
                case 'n': text += '\n'; break;
                case 't': text += '\t'; break;
                case '\\': text += '\\'; break;
                case '"': text += '"'; break;
                case '0': text += '\0'; break;
                case 'x': {
                    next_char();
                    char hex[3] = {peek_char(), 0, 0};
                    next_char();
                    if (peek_char() && std::isxdigit(peek_char())) {
                        hex[1] = peek_char();
                        next_char();
                    }
                    text += (char)std::strtol(hex, nullptr, 16);
                    continue;
                }
                default: text += peek_char(); break;
            }
            next_char();
        } else {
            text += next_char();
        }
    }
    if (peek_char() == '"') next_char(); // skip closing "
    Token t;
    t.type = TokenType::STRING;
    t.lexeme = text;
    t.line = sl; t.col = sc;
    return t;
}

Token Lexer::read_identifier_or_keyword(char first) {
    size_t sl = line_, sc = col_ - 1;
    std::string text(1, first);
    while (std::isalnum(peek_char()) || peek_char() == '_') text += next_char();
    auto it = keywords_.find(text);
    TokenType type = (it != keywords_.end()) ? it->second : TokenType::IDENTIFIER;
    Token t;
    t.type = type;
    t.lexeme = text;
    t.line = sl; t.col = sc;
    return t;
}

Token Lexer::make_token(TokenType type, std::string lexeme) {
    Token t;
    t.type = type;
    t.lexeme = std::move(lexeme);
    t.line = line_; t.col = col_;
    t.int_value = 0;
    return t;
}

Token Lexer::error_token(std::string msg) {
    Token t;
    t.type = TokenType::ERROR;
    t.lexeme = std::move(msg);
    t.line = line_; t.col = col_;
    t.int_value = 0;
    return t;
}

Token Lexer::read_token() {
    skip_whitespace_and_comments();
    char c = peek_char();
    if (c == '\0') return make_token(TokenType::EOF_);

    // Now consume the first character
    next_char();

    switch (c) {
        case '(': return make_token(TokenType::LPAREN, "(");
        case ')': return make_token(TokenType::RPAREN, ")");
        case '{': return make_token(TokenType::LBRACE, "{");
        case '}': return make_token(TokenType::RBRACE, "}");
        case '[': return make_token(TokenType::LBRACKET, "[");
        case ']': return make_token(TokenType::RBRACKET, "]");
        case ';': return make_token(TokenType::SEMICOLON, ";");
        case ':':
            if (peek_char() == ':') { next_char(); return make_token(TokenType::COLON2, "::"); }
            return make_token(TokenType::COLON, ":");
        case ',': return make_token(TokenType::COMMA, ",");
        case '.':
            if (peek_char() == '.') {
                next_char();
                if (peek_char() == '.') { next_char(); return make_token(TokenType::ELLIPSIS, "..."); }
                return make_token(TokenType::RANGE, "..");
            }
            return make_token(TokenType::DOT, ".");
        case '+': return make_token(TokenType::PLUS, "+");
        case '-':
            if (peek_char() == '>') { next_char(); return make_token(TokenType::ARROW, "->"); }
            return make_token(TokenType::MINUS, "-");
        case '*': return make_token(TokenType::STAR, "*");
        case '/': return make_token(TokenType::SLASH, "/");
        case '%': return make_token(TokenType::PERCENT, "%");
        case '=':
            if (peek_char() == '=') { next_char(); return make_token(TokenType::EQEQ, "=="); }
            return make_token(TokenType::EQ, "=");
        case '!':
            if (peek_char() == '=') { next_char(); return make_token(TokenType::NEQ, "!="); }
            return make_token(TokenType::NOT, "!");
        case '<':
            if (peek_char() == '=') { next_char(); return make_token(TokenType::LE, "<="); }
            return make_token(TokenType::LT, "<");
        case '>':
            if (peek_char() == '=') { next_char(); return make_token(TokenType::GE, ">="); }
            return make_token(TokenType::GT, ">");
        case '&':
            if (peek_char() == '&') { next_char(); return make_token(TokenType::AND, "&&"); }
            return make_token(TokenType::AMPERSAND, "&");
        case '|':
            if (peek_char() == '|') { next_char(); return make_token(TokenType::OR, "||"); }
            return make_token(TokenType::PIPE, "|");
        case '#': return make_token(TokenType::POUND, "#");
        case '^': return make_token(TokenType::CARET, "^");
        case '"': return read_string();
        case '0': case '1': case '2': case '3': case '4':
        case '5': case '6': case '7': case '8': case '9':
            return read_number(c);
        default:
            if (std::isalpha(c) || c == '_') return read_identifier_or_keyword(c);
            return error_token(std::string("unexpected character: '") + c + "'");
    }
}

void Lexer::ensure_lookahead(size_t n) {
    while (lookahead_.size() < n)
        lookahead_.push_back(read_token());
}

Token Lexer::advance() {
    Token prev = current_;
    if (!lookahead_.empty()) {
        current_ = lookahead_.front();
        lookahead_.pop_front();
    } else {
        current_ = read_token();
    }
    return prev;
}

Token Lexer::peek(size_t n) {
    ensure_lookahead(n);
    return lookahead_[n - 1];
}