src/Scanner.cpp - master - xverizex/ximas

src/Scanner.cpp
93 строки2 KB
Dmitry
init
14 июл 2026, 01:11
14 июл 2026, 01:1167983e5
#include <cstdint>
#include <vector>
#include <iostream>

#include "Scanner.h"
#include "Code.h"
#include "Token.h"
#include "UTF8.h"

Scanner *parseCode (Code *code)
{
    Scanner *scanner = new Scanner ();
    uint64_t sz = code->buf.size ();
    std::string &buf = code->buf;

    std::vector<char> lexem_word;

    uint32_t line = 1;
    uint32_t posStart = 0;
    uint32_t posEnd = 0;

    std::vector<Token *> &listToken = scanner->listToken ();

    for (uint64_t i = 0; i < sz;) {
        
        uint64_t char_sz = utf8CharLength (buf[i]);
        if ((char_sz == 1) && isTokenDelimiter (buf[i])) {
            if (lexem_word.size () > 0) {
                posEnd--;
                lexem_word.push_back ('\0');
                Token *token = getTokenFromWord (lexem_word, posStart, posEnd, line);
                listToken.push_back (token);
            }

            posStart = ++posEnd;

            lexem_word.clear ();

            if ((buf[i] != '\n') && (buf[i] != '\t') && (buf[i] != ' ') && (buf[i] != '\t')) {
                lexem_word.push_back (buf[i]);
                lexem_word.push_back ('\0');

                Token *token = getTokenFromWord (lexem_word, posStart, posEnd, line);
                listToken.push_back (token);

                lexem_word.clear ();

            } 
            
            if (buf[i] == '\n') {
                Token *token = new Token (ximas::TokenType::END);
                listToken.push_back (token);
                line++;
            }
                
            

            posStart = ++posEnd;

            
            i += char_sz;


            continue;
        }

        uint64_t sz_for_buf = char_sz + i;
        for (int k = i; k < sz_for_buf; k++) {
            lexem_word.push_back (buf[k]);
        }

        posEnd++;

        i += char_sz;
    }

    Token *token = new Token (ximas::TokenType::END);
    listToken.push_back (token);

    return scanner;
}

void Scanner::printTokens ()
{
    size_t sz = listToken_.size ();
    for (size_t i = 0; i < sz; i++) {
        std::cout << "token: " << listToken_[i]->buf () << ": " <<
            listToken_[i]->posStart () << ": " <<
            listToken_[i]->posEnd () << ": " <<
            listToken_[i]->line () << ";" << 
            std::endl;
    }
}