loom

1
/*
2
MIT License
3

4
Copyright (c) 2021 МГТУ им. Н.Э. Баумана, кафедра ИУ-6, Михаил Фетисов,
5

6
https://bmstu.codes/lsx/simodo
7
*/
8

9
#include "simodo/inout/token/Tokenizer.h"
10

11
#include <cassert>
12

13

14
namespace
15
{
16
    std::vector<simodo::inout::Tokenizer::_NumberMask> makeInnerMask(std::vector<simodo::inout::NumberMask> mask_set)
17
    {
18
        std::vector<simodo::inout::Tokenizer::_NumberMask> result;
19

20
        for (const simodo::inout::NumberMask & mask : mask_set) {
21
            if (mask.chars == simodo::inout::BUILDING_NUMBER)
22
            {
23
                int sh = result.size();
24

25
    /* 00 */    result.push_back({u"N", mask.type, simodo::inout::TokenQualification::Integer, mask.system, {sh+1,sh+3,sh+4}, true, true});
26
    /* 01 */    result.push_back({u".", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {sh+2,sh+3,sh+4}, false, true});
27
    /* 02 */    result.push_back({u"N", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {sh+3,sh+4}, false, true});
28
    /* 03 */    result.push_back({u"e", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {sh+5,sh+6,sh+7}, false, false});
29
    /* 04 */    result.push_back({u"E", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {sh+5,sh+6,sh+7}, false, false});
30
    /* 05 */    result.push_back({u"+", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {sh+7}, false, false});
31
    /* 06 */    result.push_back({u"-", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {sh+7}, false, false});
32
    /* 07 */    result.push_back({u"N", mask.type, simodo::inout::TokenQualification::RealNumber, mask.system, {}, false, true});
33
            }
34
            else
35
                result.push_back({mask.chars, mask.type, simodo::inout::TokenQualification::Integer, mask.system, {}, true, true});
36
        }
37

38
        return result;
39
    }
40
}
41

42
namespace simodo::inout
43
{
44

45
Tokenizer::Tokenizer(uri_index_t uri_index,
46
                     InputStream_interface &input_stream,
47
                     const LexicalParameters & parameters,
48
                     context_index_t context_no)
49
    : _scanner(uri_index,input_stream,context_no)
50
    , _param(parameters)
51
{
52
    assert(!_param.digits.empty());
53
    assert(!_param.latin_alphabet.empty());
54
    assert(_param.latin_alphabet.size()%2 == 0);
55
    assert(_param.national_alphabet.size()%2 == 0);
56

57
    /// \note Для отладки
58
    // parameters.masks.push_back({ u"0[1[2]][{3[4]|5[6]}[{{7}[8]}]9]", LexemeType::Number, 10 });
59
    // parameters.masks.push_back({ u"123{4|5}", LexemeType::Number, 10 });
60

61
    _numbers = makeInnerMask(parameters.masks);
62
}
63

64
Token Tokenizer::getToken()
65
{
66
    Token t = getAnyToken();
67

68
    while(t.type() == LexemeType::Comment)
69
        t = getAnyToken();
70

71
    return t;
72
}
73

74
Token Tokenizer::getAnyToken()
75
{
76
    // Конец потока
77
    if (_scanner.eof()) {
78
        return { LexemeType::Empty, u"", _scanner.makeTokenLocation(), TokenQualification::None, _scanner.context() };
79
    }
80

81
    // Проверяем контекст
82
    if (_scanner.context() != NO_TOKEN_CONTEXT_INDEX)
83
    {
84
        context_index_t context_index = _scanner.context();
85

86
        _scanner.fixLocation(context_index);
87
        return scanMarkup(context_index);
88
    }
89

90
    // Пропускаем пробелы
91
    passBlanks();
92

93
    // Фиксируем координаты начала токена
94
    _scanner.fixLocation();
95

96
    if (_scanner.eof()) {
97
        return { LexemeType::Empty, u"", _scanner.makeTokenLocation() };
98
    }
99

100
    // Символ новой строки может быть заменён на спец символ
101
    if (!_param.nl_substitution.empty() && _scanner.getChar() == '\n') {
102
        _scanner.shift(1);
103
        return { LexemeType::NewLine, _param.nl_substitution, _scanner.makeTokenLocation() };
104
    }
105

106
    // Маркированный текст
107
    for(size_t i=0; i < _param.markups.size(); ++i) {
108
        const MarkupSymbol & mus = _param.markups[i];
109

110
        if (_scanner.startsWith(mus.start))
111
            return scanMarkup(static_cast<uint32_t>(i));
112
    }
113

114
    // Слово (переменная, идентификатор и пр.)
115
    if (_scanner.startsWithAnyOf(_param.id_extra_symbols))
116
        return scanWord(NationalCharAffiliation::Extra);
117

118
    // Слово (переменная, идентификатор и пр.)
119
    if (_scanner.startsWithAnyOf(_param.latin_alphabet))
120
        return scanWord(NationalCharAffiliation::Latin);
121

122
    // Слово (переменная, идентификатор и пр.)
123
    if (_scanner.startsWithAnyOf(_param.national_alphabet))
124
        return scanWord(NationalCharAffiliation::National);
125

126
    // Число?
127
    {
128
        LexemeType          type;
129
        TokenQualification  qualification;
130
        std::u16string      lexeme_str;
131

132
        if (scanNumber(type,qualification,lexeme_str))
133
            return { type, lexeme_str, _scanner.makeTokenLocation(), qualification };
134
    }
135

136
    // Многосимвольная пунктуация (не ключевое слово)
137
    for(const std::u16string & s : _param.punctuation_words)
138
        if (_scanner.startsWith(s))
139
        {
140
            if (s == _param.eof_symbol)
141
                _scanner.setEOF();
142

143
            _scanner.shift(s.size());
144

145
            return { LexemeType::Punctuation, s, _scanner.makeTokenLocation() };
146
        }
147

148
    // Односимвольная пунктуация
149
    if (_scanner.startsWithAnyOf(_param.punctuation_chars))
150
    {
151
        std::u16string s;
152
        s.assign(1, _scanner.getFirstChar());
153

154
        if (s == _param.eof_symbol)
155
            _scanner.setEOF();
156

157
        _scanner.shift(1);
158

159
        return { LexemeType::Punctuation, s, _scanner.makeTokenLocation() };
160
    }
161

162
    // Что-то неизвестное, т.е. ошибка
163
    std::u16string lexeme_str;
164

165
    lexeme_str += _scanner.getFirstChar();
166
    _scanner.shift(1);
167

168
    return { LexemeType::Error, lexeme_str, _scanner.makeTokenLocation(), TokenQualification::UnknownCharacterSet };
169
}
170

171
void Tokenizer::passBlanks()
172
{
173
    while(!_scanner.eof())
174
    {
175
        if (!_param.nl_substitution.empty() && _scanner.getChar() == '\n')
176
            break;
177
        if (!isBlank(_scanner.getChar()))
178
            break;
179

180
        _scanner.shift(1);
181
    }
182
}
183

184
Token Tokenizer::scanMarkup(context_index_t context)
185
{
186
    assert(context < _param.markups.size());
187

188
    std::u16string lexeme_str;
189
    std::u16string token_str;
190

191
    const MarkupSymbol & mus = _param.markups[context];
192

193
    if (_scanner.context() != context)
194
    {
195
        _scanner.shift(mus.start.size());
196
        token_str += mus.start;
197
    }
198

199
    while(!_scanner.eof())
200
    {
201
        if (_scanner.startsWith(mus.ignore_sign))
202
        {
203
            token_str += mus.ignore_sign;
204
            _scanner.shift(mus.ignore_sign.size());
205

206
            if (_scanner.eof())
207
                break;
208

209
            token_str += _scanner.getFirstChar();
210
            lexeme_str += _scanner.getFirstChar();
211
            _scanner.shift(1);
212
        }
213
        else
214
        {
215
            if (mus.end.empty())
216
            {
217
                if (_scanner.getFirstChar() == u'\n')
218
                    break;
219
            }
220
            else if (_scanner.startsWith(mus.end))
221
            {
222
                token_str += mus.end;
223
                _scanner.shift(mus.end.size());
224
                context = NO_TOKEN_CONTEXT_INDEX;
225
                break;
226
            }
227

228
            token_str += _scanner.getFirstChar();
229
            lexeme_str += _scanner.getFirstChar();
230
            _scanner.shift(1);
231
        }
232
    }
233

234
    if (mus.end.empty())
235
        context = NO_TOKEN_CONTEXT_INDEX;
236

237
    TokenLocation loc = _scanner.makeTokenLocation();
238

239
    _scanner.fixLocation(context);
240

241
    return { {lexeme_str, mus.type}, token_str, loc, TokenQualification::None, context };
242
}
243

244
Token Tokenizer::scanWord(Tokenizer::NationalCharAffiliation first_char)
245
{
246
    bool    has_latin    = (first_char == NationalCharAffiliation::Latin);
247
    bool    has_national = (first_char == NationalCharAffiliation::National);
248

249
    std::u16string lexeme_str;
250

251
    lexeme_str += _scanner.getFirstChar();
252
    _scanner.shift(1);
253

254
    // Формируем лексему
255
    while(!_scanner.eof())
256
    {
257
        if (_scanner.startsWithAnyOf(_param.id_extra_symbols))
258
            ;
259
        else if (_scanner.startsWithAnyOf(_param.latin_alphabet))
260
            has_latin = true;
261
        else if (_scanner.startsWithAnyOf(_param.national_alphabet))
262
            has_national = true;
263
        else if (!_scanner.startsWithAnyOf(_param.digits))
264
            break;
265

266
        lexeme_str += _scanner.getFirstChar();
267
        _scanner.shift(1);
268
    }
269

270
    // Многосимвольная пунктуация
271
    for(const std::u16string & s : _param.punctuation_words)
272
    {
273
        bool is_find;
274

275
        if (_param.is_case_sensitive)
276
            is_find = (s == lexeme_str);
277
        else
278
            is_find = (s == convertToUpper(lexeme_str));
279

280
        if (is_find)
281
        {
282
            if (s == _param.eof_symbol)
283
                _scanner.setEOF();
284

285
            if (_param.is_case_sensitive)
286
                return { {lexeme_str, LexemeType::Punctuation}, s, _scanner.makeTokenLocation(), TokenQualification::Keyword };
287

288
            return { LexemeType::Punctuation, lexeme_str, _scanner.makeTokenLocation(), TokenQualification::Keyword };
289
        }
290
    }
291

292
    // Односимвольная пунктуация
293
    if (lexeme_str.size() == 1)
294
        if (_param.punctuation_chars.find(*lexeme_str.c_str()) != std::u16string::npos)
295
            return { LexemeType::Punctuation, lexeme_str, _scanner.makeTokenLocation(), TokenQualification::Keyword};
296

297
    if (has_national)
298
    {
299
        if (!_param.may_national_letters_use)
300
            return { LexemeType::Error, lexeme_str, _scanner.makeTokenLocation(), TokenQualification::NationalCharacterUse };
301

302
        if (has_latin)
303
            return { _param.may_national_letters_mix ? LexemeType::Id : LexemeType::Error,
304
                     lexeme_str, _scanner.makeTokenLocation(), TokenQualification::NationalCharacterMix };
305
    }
306

307
    return { LexemeType::Id, lexeme_str, _scanner.makeTokenLocation() };
308
}
309

310
bool Tokenizer::scanNumber(LexemeType &type, TokenQualification &qualification, std::u16string &lexeme_str)
311
{
312
    for(size_t i_starting=0; i_starting < _numbers.size(); ++i_starting)
313
    {
314
        if (_numbers[i_starting].is_starting)
315
        {
316
            lexeme_str.clear();
317

318
            size_t  i_mask_index = i_starting;
319
            size_t  i_mask_char  = 0;
320
            size_t  i_input      = 0;
321
            int16_t N_count      = -1;
322

323
            while(true)
324
            {
325
                const _NumberMask & mask    = _numbers[i_mask_index];
326
                char16_t            ch      = _scanner.getChar(i_input);
327

328
                if (i_mask_char == mask.chars.size())
329
                {
330
                    if (lexeme_str.empty())
331
                        return false;
332

333
                    if (mask.chars[i_mask_char-1] == u'n')
334
                    {
335
                        char16_t ch_upper;
336
                        if (mask.system > 10 )
337
                            ch_upper = convertLatinToUpper(ch);
338
                        else
339
                            ch_upper = ch;
340

341
                        if (_param.digits.find(ch_upper) < mask.system)
342
                            break;
343
                    }
344

345
                    if (!mask.refs.empty())
346
                    {
347
                        size_t i_ref=0;
348
                        for(; i_ref < mask.refs.size(); ++i_ref)
349
                        {
350
                            uint8_t ref_no = mask.refs[i_ref];
351

352
                            assert(static_cast<size_t>(ref_no) < _numbers.size());
353

354
                            const _NumberMask & ref_mask = _numbers[static_cast<size_t>(ref_no)];
355

356
                            assert(!ref_mask.chars.empty());
357

358
                            char16_t ch_upper;
359

360
                            if (mask.system > 10 )
361
                                ch_upper = convertLatinToUpper(ch);
362
                            else
363
                                ch_upper = ch;
364

365
                            if (ch == ref_mask.chars[0]
366
                             || ((ref_mask.chars[0] == u'N' || ref_mask.chars[0] == u'n') && _param.digits.find(ch_upper) < mask.system))
367
                                break;
368
                        }
369

370
                        if (i_ref < mask.refs.size())
371
                        {
372
                            i_mask_index = static_cast<size_t>(mask.refs[i_ref]);
373
                            i_mask_char = 0;
374
                            continue;
375
                        }
376

377
                        if (!mask.may_final)
378
                            break;
379
                    }
380

381
                    type = mask.type;
382
                    qualification = mask.qualification;
383
                    _scanner.shift(lexeme_str.size());
384
                    return true;
385
                }
386

387
                if (mask.chars[i_mask_char] == u'N' || mask.chars[i_mask_char] == u'n')
388
                {
389
                    N_count ++;
390

391
                    char16_t ch_upper;
392
                    if (mask.system > 10 )
393
                        ch_upper = convertLatinToUpper(ch);
394
                    else
395
                        ch_upper = ch;
396

397
                    if (_param.digits.find(ch_upper) < mask.system)
398
                    {
399
                        lexeme_str += ch;
400
                        if (mask.chars[i_mask_char] == u'n')
401
                            i_mask_char ++;
402
                    }
403
                    else if (N_count == 0)
404
                        break;
405
                    else if (mask.chars[i_mask_char] == u'n')
406
                        break;
407
                    else
408
                    {
409
                        i_mask_char ++;
410
                        continue;
411
                    }
412
                }
413
                else if (mask.chars[i_mask_char] == ch)
414
                {
415
                    lexeme_str += ch;
416
                    i_mask_char ++;
417
                    N_count = -1;
418
                }
419
                else
420
                    break;
421

422
                i_input ++ ;
423
            }
424
        }
425
        // don't remove!
426
        else
427
            break;
428
    }
429

430
    if (!lexeme_str.empty())
431
    {
432
        _scanner.shift(lexeme_str.size());
433
        type = LexemeType::Error;
434
        qualification = TokenQualification::NotANumber;
435
        return true;
436
    }
437

438
    return false;
439
}
440

441
std::u16string Tokenizer::convertToUpper(std::u16string s) const
442
{
443
    std::u16string res;
444

445
    for(char16_t c : s)
446
        res += convertToUpper(c);
447

448
    return res;
449
}
450

451
char16_t Tokenizer::convertToUpper(char16_t ch) const
452
{
453
    std::string::size_type  pos_latin       = _param.latin_alphabet.find(ch);
454
    size_t                  latin_size      = _param.latin_alphabet.size();
455
    size_t                  national_size   = _param.national_alphabet.size();
456

457
    if (pos_latin != std::string::npos)
458
    {
459
        if (pos_latin < latin_size/2)
460
            return _param.latin_alphabet.at(latin_size/2+pos_latin);
461

462
        return ch;
463
    }
464

465
    std::string::size_type pos_national = _param.national_alphabet.find(ch);
466

467
    if (pos_national != std::string::npos)
468
        if (pos_national < national_size/2)
469
            return _param.national_alphabet.at(national_size/2+pos_national);
470

471
    return ch;
472
}
473

474
char16_t Tokenizer::convertLatinToUpper(char16_t ch) const
475
{
476
    std::string::size_type  pos_latin       = _param.latin_alphabet.find(ch);
477
    size_t                  latin_size      = _param.latin_alphabet.size();
478

479
    if (pos_latin != std::string::npos)
480
        if (pos_latin < latin_size/2)
481
            return _param.latin_alphabet.at(latin_size/2+pos_latin);
482

483
    return ch;
484
}
485

486
bool Tokenizer::isBlank(char16_t ch) const
487
{
488
    return (std::u16string::npos != std::u16string(u" \t\r\n").find(ch));
489
}
490

491
}