tokenizer.cpp - master - Boxapp/yadisk-helper

tokenizer.cpp
367 строк8 KB
DebianCougar
test build 2026.02-beta
30 янв 2026, 21:25
30 янв 2026, 21:25eb3551d
#include <stdexcept>

#include "tokenizer.hpp"

// TODO throw specific exception instead of STL exceptions

enum CHAR_TYPE {
	ASTERISK, BACKSLASH, NAME, NEWLINE, OPERATOR, OTHER, QUOTE, SEMICOLON, SHARP, SINGLE_QUOTE, SLASH, SPACE
};

enum STATE {
	CIC, CIE, CL, CS, N, O, S, TEC, TEE, TR, TUC, TUE
};

static std::vector<bool> getCharMarks(const std::string &tomark) {
	std::vector<bool> marks(256, false);
	for (char c : tomark) marks[c] = true;
	return marks;
}

static const std::string extraname = "-.";
static const std::vector<bool> allowed = getCharMarks(extraname);

static CHAR_TYPE char_type(char c) {
	switch (c) {
		case ' ': case '\t': return SPACE;
		case '\n': case '\r': return NEWLINE;
		case '/': return SLASH;
		case '*': return ASTERISK;
		case '#': return SHARP;
		case ';': return SEMICOLON;
		case '\'': return SINGLE_QUOTE;
		case '"': return QUOTE;
		case '\\': return BACKSLASH;
		default:
			if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c == '_')) return NAME;
			if (c > ' ' && c <= '~') return allowed[c] ? NAME : OPERATOR;
			return OTHER;
	}
}

Config tokenize(std::istream &input/* = std::cin*/) {
	Config res;
	STATE state = N;
	char c = '\0';
	Line line;
	TOKEN token;
	bool filled = false;
	while (input.get(c)) {
		CHAR_TYPE t = char_type(c);
		switch (state) { // first state then char
			case CIC:
				if (t == ASTERISK) state = CIE;
				break;
			case CIE:
				if (t == SLASH) state = S;
				else if (t != ASTERISK) state = CIC;
				break;
			case CL:
				if (t == NEWLINE) state = N;
				break;
			case CS:
				switch (t) {
					case ASTERISK:
						if (filled) {
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						state = CIC;
						break;
					case BACKSLASH:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
						}
						token.type = NAME_TOKEN;
						token.value.clear();
						filled = true;
						state = TUE;
						break;
					case NAME:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
						}
						token.type = NAME_TOKEN;
						token.value.clear();
						token.value += c;
						filled = true;
						state = TUC;
						break;
					case NEWLINE: case SEMICOLON:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						if (!line.empty()) {
							res.push_back(line);
							line.clear();
						}
						state = N;
						break;
					case OPERATOR:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
						}
						token.type = OPER_TOKEN;
						token.value += c;
						filled = true;
						state = O;
						break;
					case OTHER: throw std::invalid_argument("Unexpected character with code "+std::to_string(c));
					case QUOTE:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
						}
						token.type = NAME_TOKEN;
						token.value.clear();
						filled = true;
						state = TEC;
						break;
					case SHARP:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						if (!line.empty()) {
							res.push_back(line);
							line.clear();
						}
						state = CL;
						break;
					case SINGLE_QUOTE:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
						}
						token.type = NAME_TOKEN;
						token.value.clear();
						filled = true;
						state = TR;
						break;
					case SLASH:
						if (filled) {
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						if (!line.empty()) {
							res.push_back(line);
							line.clear();
						}
						state = CL;
						break;
					case SPACE:
						if (filled) {
							if (token.type == OPER_TOKEN) token.value += '/';
							else {
								line.push_back(token);
								token.type = OPER_TOKEN;
								token.value = "/";
							}
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						state = S;
						break;
					default: break;
				}
				break;
			case N: case O: case S: case TUC:
				switch (t) {
					case ASTERISK: case OPERATOR:
						if (filled && token.type == NAME_TOKEN) {
							line.push_back(token);
							token.value.clear();
						}
						token.type = OPER_TOKEN;
						token.value += c;
						filled = true;
						state = O;
						break;
					case BACKSLASH:
						if (filled && token.type == OPER_TOKEN) {
							line.push_back(token);
							token.value.clear();
						}
						token.type = NAME_TOKEN;
						filled = true;
						state = TUE;
						break;
					case NAME:
						if (filled && token.type == OPER_TOKEN) {
							line.push_back(token);
							token.value.clear();
						}
						token.type = NAME_TOKEN;
						token.value += c;
						filled = true;
						state = TUC;
						break;
					case NEWLINE: case SEMICOLON:
						if (filled) {
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						if (!line.empty()) {
							res.push_back(line);
							line.clear();
						}
						state = N;
						break;
					case OTHER: throw std::invalid_argument("Unexpected character with code "+std::to_string(c));
					case QUOTE:
						if (filled && token.type == OPER_TOKEN) {
							line.push_back(token);
							token.value.clear();
						}
						token.type = NAME_TOKEN;
						filled = true;
						state = TEC;
						break;
					case SHARP:
						if (filled) {
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						if (!line.empty()) {
							res.push_back(line);
							line.clear();
						}
						state = CL;
						break;
					case SINGLE_QUOTE:
						if (filled && token.type == OPER_TOKEN) {
							line.push_back(token);
							token.value.clear();
						}
						token.type = NAME_TOKEN;
						filled = true;
						state = TR;
						break;
					case SLASH:
						if (filled && token.type == NAME_TOKEN) {
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						state = CS;
						break;
					case SPACE:
						if (filled) {
							line.push_back(token);
							token.value.clear();
							filled = false;
						}
						state = S;
						break;
					default: break;
				}
				break;
			case TEC:
				if (t == QUOTE) state = TUC;
				else if (t == BACKSLASH) state = TEE;
				else {
					token.type = NAME_TOKEN;
					token.value += c;
					filled = true;
				}
				break;
			case TEE:
				token.value += c;
				state = TEC;
				break;
			case TR:
				if (t == SINGLE_QUOTE) state = TUC;
				else {
					token.type = NAME_TOKEN;
					token.value += c;
					filled = true;
				}
				break;
			case TUE:
				token.value += c;
				state = TUC;
				break;
			default: break;
		}
	}
	switch (state) { // char_type == EOF
		case CIC: case CIE:
			if (!line.empty()) {
				res.push_back(line);
				line.clear();
			}
			break;
		case CS:
			if (filled && token.type == NAME_TOKEN) {
				line.push_back(token);
				token.value.clear();
			}
			token.type = OPER_TOKEN;
			token.value += '/';
			line.push_back(token);
			token.value.clear();
			res.push_back(line);
			line.clear();
			break;
		case O: case S: case TUC:
			if (filled) {
				line.push_back(token);
				token.value.clear();
			}
			if (!line.empty()) {
				res.push_back(line);
				line.clear();
			}
			break;
		case TEC: case TEE: case TR: case TUE: throw std::length_error("Unexpected end of input");
		default: break;
	}
	return res;
}