loom

1
/*
2
MIT License
3

4
Copyright (c) 2021 МГТУ им. Н.Э. Баумана, кафедра ИУ-6, Михаил Фетисов,
5

6
https://bmstu.codes/lsx/simodo
7
*/
8

9
#include "simodo/inout/token/InputStream.h"
10

11
#include <cassert>
12
#include <cstdint>
13

14
namespace
15
{
16
    template <int size>
17
    struct UTF_BOM
18
    {
19
        const unsigned char bytes[size];
20
    };
21

22
    static constexpr UTF_BOM<3> UTF8_BOM = {
23
        {0xEF, 0xBB, 0xBF}
24
    };
25
    static constexpr UTF_BOM<2> UTF16BE_BOM = {
26
        {0xFE, 0xFF}
27
    };
28
    static constexpr UTF_BOM<2> UTF16LE_BOM = {
29
        {0xFF, 0xFE}
30
    };
31
    static constexpr UTF_BOM<4> UTF32BE_BOM = {
32
        {0x00, 0x00, 0xFE, 0xFF}
33
    };
34
    static constexpr UTF_BOM<4> UTF32LE_BOM = {
35
        {0xFF, 0xFE, 0x00, 0x00}
36
    };
37

38
    template <int bom_size>
39
    static bool skipUtfBom(
40
        std::istream & is
41
        , const UTF_BOM<bom_size> bom)
42
    {    
43
        int i = 0;
44
        for (
45
            ; is.peek() == bom.bytes[i] && i < bom_size
46
            ; is.get(), ++i
47
        );
48

49
        if (i == bom_size) return true;
50

51
        for (
52
            ; 0 < i
53
            ; is.unget(), --i
54
        );
55

56
        return false;
57
    }
58
}
59

60
namespace simodo::inout
61
{
62

63
char16_t InputStream::get()
64
{
65
    if (_surrogate_pair != 0)
66
    {
67
        char16_t ch = _surrogate_pair;
68
        _surrogate_pair = 0;
69
        return ch;
70
    }
71

72
    if (_in.eof())
73
        return std::char_traits<char16_t>::eof();
74

75
    if (_untouched)
76
    {
77
        skipUtfBom(_in, UTF8_BOM)
78
            || skipUtfBom(_in, UTF16BE_BOM)
79
            || skipUtfBom(_in, UTF16LE_BOM)
80
            || skipUtfBom(_in, UTF32BE_BOM)
81
            || skipUtfBom(_in, UTF32LE_BOM);
82

83
        _untouched = false;
84
    }
85

86
    int ch1 = _in.get();
87

88
    if (ch1 <= 0x7F)
89
        return static_cast<char16_t>(ch1);
90

91
    uint32_t code;
92
    int      count;
93

94
    if ((ch1 & 0b11100000) == 0b11000000)
95
    {
96
        count = 2;
97
        code = (ch1 & 0b00011111);
98
    }
99
    else if ((ch1 & 0b11110000) == 0b11100000)
100
    {
101
        count = 3;
102
        code = (ch1 & 0b00001111);
103
    }
104
    else
105
    {
106
        count = 4;
107
        code = (ch1 & 0b00000111);
108
    }
109

110
    for(int i=1; i < count; ++i)
111
    {
112
        int ch = _in.get();
113

114
        if (ch == std::char_traits<char16_t>::eof())
115
            break;
116

117
        code = (code << 6) + (static_cast<uint32_t>(ch) & 0b00111111);
118
    }
119

120
    if (code <= 0xD7FF ||
121
       ((code >= 0xE000) && (code <= 0xFFFF)) )
122
    {
123
        return static_cast<char16_t>(code);
124
    }
125
    else if ((code >= 0xD800) && (code <= 0xDFFF))
126
    {
127
        // unicode replacement character
128
        return 0xFFFD;
129
    }
130
    else
131
    {
132
        // surrogate pair
133
        code -= 0x010000;
134
        _surrogate_pair = 0xD800 + ((code >> 10) & 0x3FF);
135
        return 0xDC00 + (code & 0x3FF);
136
    }
137
}
138

139
}