lib/Services/TokensParser.cpp - main - thomas-king/SyntacticAnalyzer

lib/Services/TokensParser.cpp
440 строк10 KB
Ace Rodstin
Add ParseExpressionTests. Fix parenthesis parsing.
06 апр 2023, 13:52
06 апр 2023, 13:52760e23e
//
//  TokensParser.cpp
//  TokensParser
//
//  Created by Ace Rodstin on 3/3/23.
//  Copyright © 2023 Ace Rodstin. All rights reserved.
//

#include "Services/TokensParser.h"

using namespace ace;

TokensParser::TokensParser(const string buffer) {
	this->currentPosition = buffer.begin();
	this->end = buffer.end();
}

Token TokensParser::next() {
	advanceBySpaces();
	auto character = *currentPosition;

	if (currentPosition == end || characterManager.isNull(character)) {
		return { Token::Kind::endOfInput };
	}

	optional<Token> token;

	if (characterManager.isLetter(character) || character == '_') {
		token = formTextToken();
	} else if (characterManager.isDigit(character, NumberBase::decimal)) {
		token = formNumberToken();
	} else if (auto operatorToken = formOperatorToken()) {
		token = operatorToken;
	} else if (auto punctuatorToken = formPunctuatorToken()) {
		token = punctuatorToken;
	} else if (character == '\"') {
		token = formStringToken();
	} else if (character == '\'') {
		token = formCharacterToken();
	}

	if (token.has_value()) {
		return token.value();
	} else {
		return { Token::Kind::unknown };
	}
}

optional<Token> TokensParser::formTextToken() {
	iterator tokenEnd = currentPosition + 1;
	auto character = *tokenEnd;

	while (characterManager.isLetter(character) || character == '_' || characterManager.isDigit(character, NumberBase::decimal)) {
		++tokenEnd;
		character = *tokenEnd;
	}

	auto _text = text(tokenEnd);
	advanceBy(_text);

	if (auto _keyword = keyword(_text)) {
		return Token { _keyword.value() };
	} else {
		return Token { _text };
	}
}

optional<Token> TokensParser::formNumberToken() {
	auto character = *currentPosition;
	auto nextCharacter = *(currentPosition + 1);

	optional<Literal> literal;

	if (character == '0') {
		switch (nextCharacter) {
		case 'b':
			literal = parseIntegerLiteral(NumberBase::binary);
			break;
		case 'o':
			literal = parseIntegerLiteral(NumberBase::octal);
			break;
		case 'x':
			literal = parseHexadecimalNumberLiteral();
			break;
		default:
			literal = parseDecimalNumberLiteral();
			break;
		}
	} else {
		literal = parseDecimalNumberLiteral();
	}

	if (literal.has_value()) {
		return Token { literal.value() };
	} else {
		return Token { Token::Kind::unknown };
	}
}

optional<Token> TokensParser::formOperatorToken() {
	auto isValid = [&](char character) -> bool {
		auto isDelimiter = characterManager.isDelimiter(character);
		auto isDigit = characterManager.isDigit(character, NumberBase::decimal);
		auto isLetter = characterManager.isLetter(character);
		return !(isDelimiter || isDigit || isLetter);
	};

	auto tokenEnd = currentPosition + 1;
	auto character = *tokenEnd;

	while (isValid(character)) {
		++tokenEnd;
		character = *tokenEnd;
	}

	auto _text = text(tokenEnd);

	if (auto __operator = _operator(_text)) {
		advanceBy(__operator.value());
		return Token { __operator.value() };
	} else {
		return {};
	}
}

optional<Token> TokensParser::formPunctuatorToken() {
	auto tokenEnd = currentPosition + 1;
	auto _text = text(tokenEnd);

	if (auto _punctuator = punctuator(_text)) {
		advanceBy(_punctuator.value());
		return Token { _punctuator.value() };
	} else {
		return {};
	}
}

optional<Token> TokensParser::formStringToken() {
	advanceBy(1); // Skip opening quotes mark

	iterator tokenEnd = currentPosition + 1;
	auto character = *tokenEnd;

	while (character != '\"') {
		++tokenEnd;
		character = *tokenEnd;
	}

	auto _text = text(tokenEnd);
	advanceBy(_text);
	advanceBy(1); // Skip closing quotes mark

	Literal literal { Literal::Kind::string, _text };
	return Token { literal };
}

optional<Token> TokensParser::formCharacterToken() {
	advanceBy(1); // Skip opening quote mark

	iterator tokenEnd = currentPosition + 1;
	auto character = *tokenEnd;

	while (character != '\'') {
		++tokenEnd;
		character = *tokenEnd;
	}

	auto _text = text(tokenEnd);
	advanceBy(_text);
	advanceBy(1); // Skip closing quote mark

	if (_text.length() != 1) {
		return {};
	} else {
		Literal literal { Literal::Kind::character, _text };
		return Token { literal };
	}
}

optional<Literal> TokensParser::parseDecimalNumberLiteral() {
	optional<Literal> literal;
	NumberBase base = NumberBase::decimal;

	auto decimalDigit = text(currentPosition + 1);
	advanceBy(decimalDigit);

	auto character = *currentPosition;
	auto isValid = [&](char character) -> bool {
		auto rawValue = string(1, character);

		return punctuator(rawValue)
			|| _operator(rawValue)
			|| characterManager.isExponent(character, base)
			|| characterManager.isDelimiter(character);
	};

	if (characterManager.isDigit(character, NumberBase::decimal)) {
		if (auto decimalDigits = parseNumberDigits(base)) {
			auto head = decimalDigit + decimalDigits.value();
			literal =  parseNumberLiteral(head, base);
		}
	} else if (character == '_') {
		auto underscore = text(currentPosition + 1);
		advanceBy(underscore);

		if (auto decimalDigits = parseNumberDigits(base)) {
			auto head = decimalDigit + underscore + decimalDigits.value();
			literal = parseNumberLiteral(head, base);
		}
	} else if (isValid(character)) {
		auto head = decimalDigit;
		literal = parseNumberLiteral(head, base);
	}

	return literal;
}

optional<Literal> TokensParser::parseHexadecimalNumberLiteral() {
	optional<Literal> literal;
	NumberBase base = NumberBase::hexadecimal;

	auto head = text(currentPosition + 2);
	advanceBy(head);

	auto character = *currentPosition;

	if (characterManager.isDigit(character, NumberBase::hexadecimal)) {
		if (auto hexadecimalDigits = parseNumberDigits(base)) {
			head += hexadecimalDigits.value();
			literal =  parseNumberLiteral(head, base);
		}
	} else if (character == '_') {
		auto underscore = text(currentPosition + 1);
		advanceBy(underscore);

		if (auto hexadecimalDigits = parseNumberDigits(base)) {
			head += underscore + hexadecimalDigits.value();
			literal = parseNumberLiteral(head, base);
		}
	}

	return literal;
}

optional<Literal> TokensParser::parseNumberLiteral(string head, NumberBase base) {
	auto character = *currentPosition;

	if (character == '.') {
		if (auto fraction = parseFraction(base)) {
			auto kind = Literal::Kind::_float;
			auto value = head + fraction.value();
			return make_optional<Literal>(kind, value, base);
		} else {
			return {};
		}
	}

	if (characterManager.isExponent(character, base)) {
		if (auto exponent = parseExponent(base)) {
			auto kind = Literal::Kind::_float;
			auto value = head + exponent.value();
			return make_optional<Literal>(kind, value, base);
		} else {
			return {};
		}
	}

	auto kind = Literal::Kind::integer;
	auto value = head;
	return make_optional<Literal>(kind, value, base);
}

optional<Literal> TokensParser::parseIntegerLiteral(NumberBase base) {
	auto character = *currentPosition;

	if (!characterManager.isDigit(character, NumberBase::decimal)) {
		return {};
	}

	auto kind = Literal::Kind::integer;
	auto tokenEnd = currentPosition + 2;
	character = *tokenEnd;

	if (character == '_') {
		++tokenEnd;	
	}

	auto _text = text(tokenEnd);
	advanceBy(_text);

	auto digits = parseNumberDigits(base);

	if (digits.has_value()) {
		auto value = _text + digits.value();
		return make_optional<Literal>(kind, value, base);
	} else {
		return {};
	}
}

optional<string> TokensParser::parseFraction(NumberBase base) {
	auto character = *currentPosition;

	if (character != '.') {
		return {};
	}

	auto dot = text(currentPosition + 1);
	advanceBy(dot);

	auto digits = parseNumberDigits(base);

	if (!digits.has_value()) {
		return {};
	}

	character = *currentPosition;

	if (character == '.') {
		return {};
	}

	auto fraction = dot + digits.value();

	if (auto exponent = parseExponent(base)) {
		return fraction + exponent.value();
	} else {
		return fraction;
	}
}

optional<string> TokensParser::parseExponent(NumberBase base) {
	auto character = *currentPosition;

	if (!characterManager.isExponent(character, base)) {
		return {};
	}

	auto exponent = text(currentPosition + 1);
	advanceBy(exponent);

	string sign = "";
	character = *currentPosition;

	if (characterManager.isSign(character)) {
		sign = text(currentPosition + 1);
		advanceBy(sign);
	}

	if (auto decimalDigits = parseNumberDigits(NumberBase::decimal)) {
		return exponent + sign + decimalDigits.value();
	} else {
		return {};
	}
}

optional<string> TokensParser::parseNumberDigits(NumberBase base) {
	auto isSeparator = [&, base](char character) -> bool {
		return characterManager.isDelimiter(character)
			|| character == '.'
			|| characterManager.isExponent(character, base);
	};

	auto isValid = [&, base](char character, char prevCharacter) -> bool {
		if (character == '_') {
			return prevCharacter != '_';
		} else {
			return characterManager.isDigit(character, base);
		}
	};

	auto character = *currentPosition;
	auto prevCharacter = character;

	if (!characterManager.isDigit(character, base)) {
		return {};
	}

	auto tokenEnd = currentPosition + 1;
	character = *tokenEnd;

	while (!isSeparator(character)) {
		if (isValid(character, prevCharacter)) {
			++tokenEnd;
			prevCharacter = character;
			character = *tokenEnd;
		} else {
			return {};
		}
	}

	auto _text = text(tokenEnd);
	advanceBy(_text);
	return _text;
}

string TokensParser::text(iterator tokenEnd) {
	iterator begin = currentPosition;
	iterator end = tokenEnd;
	return string(begin, end);
}

optional<Keyword> TokensParser::keyword(string text) {
	auto value = text;
	return findIn(keywords, value);
}

optional<Operator> TokensParser::_operator(string text) {
	auto value = text;
	return findIn(operators, value);
}

optional<Punctuator> TokensParser::punctuator(string text) {
	auto value = text;
	return findIn(punctuators, value);
}

void TokensParser::advanceBySpaces() {
	while (currentPosition != end && characterManager.isWhitespace(*currentPosition)) {
		++currentPosition;
	}
}

void TokensParser::advanceBy(string text) {
	advanceBy(text.length());
}

void TokensParser::advanceBy(Operator op) {
	auto rawValue = operators.at(op);
	advanceBy(rawValue.length());
}

void TokensParser::advanceBy(Punctuator punctuator) {
	auto rawValue = punctuators.at(punctuator);
	advanceBy(rawValue.length());
}

void TokensParser::advanceBy(int charactersCount) {
	currentPosition += charactersCount;
}