ClickHouse

1
#include <IO/ReadBuffer.h>
2
#include <IO/WriteBuffer.h>
3
#include <IO/ReadHelpers.h>
4
#include <IO/WriteHelpers.h>
5
#include <IO/ReadBufferFromString.h>
6
#include <IO/readDecimalText.h>
7
#include <Core/Field.h>
8
#include <Core/DecimalComparison.h>
9
#include <Common/FieldVisitorDump.h>
10
#include <Common/FieldVisitorToString.h>
11
#include <Common/FieldVisitorWriteBinary.h>
12

13

14
using namespace std::literals;
15

16
namespace DB
17
{
18

19
namespace ErrorCodes
20
{
21
    extern const int CANNOT_RESTORE_FROM_FIELD_DUMP;
22
    extern const int DECIMAL_OVERFLOW;
23
}
24

25
template <is_decimal T>
26
T DecimalField<T>::getScaleMultiplier() const
27
{
28
    return DecimalUtils::scaleMultiplier<T>(scale);
29
}
30

31
inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
32
{
33
    switch (static_cast<Field::Types::Which>(type))
34
    {
35
        case Field::Types::Null:
36
        {
37
            return Field();
38
        }
39
        case Field::Types::UInt64:
40
        {
41
            UInt64 value;
42
            readVarUInt(value, buf);
43
            return value;
44
        }
45
        case Field::Types::UInt128:
46
        {
47
            UInt128 value;
48
            readBinary(value, buf);
49
            return value;
50
        }
51
        case Field::Types::UInt256:
52
        {
53
            UInt256 value;
54
            readBinary(value, buf);
55
            return value;
56
        }
57
        case Field::Types::UUID:
58
        {
59
            UUID value;
60
            readBinary(value, buf);
61
            return value;
62
        }
63
        case Field::Types::IPv4:
64
        {
65
            IPv4 value;
66
            readBinary(value, buf);
67
            return value;
68
        }
69
        case Field::Types::IPv6:
70
        {
71
            IPv6 value;
72
            readBinary(value.toUnderType(), buf);
73
            return value;
74
        }
75
        case Field::Types::Int64:
76
        {
77
            Int64 value;
78
            readVarInt(value, buf);
79
            return value;
80
        }
81
        case Field::Types::Int128:
82
        {
83
            Int128 value;
84
            readBinary(value, buf);
85
            return value;
86
        }
87
        case Field::Types::Int256:
88
        {
89
            Int256 value;
90
            readBinary(value, buf);
91
            return value;
92
        }
93
        case Field::Types::Float64:
94
        {
95
            Float64 value;
96
            readFloatBinary(value, buf);
97
            return value;
98
        }
99
        case Field::Types::String:
100
        {
101
            std::string value;
102
            readStringBinary(value, buf);
103
            return value;
104
        }
105
        case Field::Types::Array:
106
        {
107
            Array value;
108
            readBinary(value, buf);
109
            return value;
110
        }
111
        case Field::Types::Tuple:
112
        {
113
            Tuple value;
114
            readBinary(value, buf);
115
            return value;
116
        }
117
        case Field::Types::Map:
118
        {
119
            Map value;
120
            readBinary(value, buf);
121
            return value;
122
        }
123
        case Field::Types::Object:
124
        {
125
            Object value;
126
            readBinary(value, buf);
127
            return value;
128
        }
129
        case Field::Types::AggregateFunctionState:
130
        {
131
            AggregateFunctionStateData value;
132
            readStringBinary(value.name, buf);
133
            readStringBinary(value.data, buf);
134
            return value;
135
        }
136
        case Field::Types::Bool:
137
        {
138
            UInt8 value;
139
            readBinary(value, buf);
140
            return bool(value);
141
        }
142
        case Field::Types::Decimal32:
143
        case Field::Types::Decimal64:
144
        case Field::Types::Decimal128:
145
        case Field::Types::Decimal256:
146
        case Field::Types::CustomType:
147
            return Field();
148
    }
149
    UNREACHABLE();
150
}
151

152
void readBinary(Array & x, ReadBuffer & buf)
153
{
154
    size_t size;
155
    UInt8 type;
156
    readBinary(type, buf);
157
    readBinary(size, buf);
158

159
    for (size_t index = 0; index < size; ++index)
160
        x.push_back(getBinaryValue(type, buf));
161
}
162

163
void writeBinary(const Array & x, WriteBuffer & buf)
164
{
165
    UInt8 type = Field::Types::Null;
166
    size_t size = x.size();
167
    if (size)
168
        type = x.front().getType();
169
    writeBinary(type, buf);
170
    writeBinary(size, buf);
171

172
    for (const auto & elem : x)
173
        Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem);
174
}
175

176
void writeText(const Array & x, WriteBuffer & buf)
177
{
178
    String res = applyVisitor(FieldVisitorToString(), Field(x));
179
    buf.write(res.data(), res.size());
180
}
181

182
void readBinary(Tuple & x, ReadBuffer & buf)
183
{
184
    size_t size;
185
    readBinary(size, buf);
186

187
    for (size_t index = 0; index < size; ++index)
188
    {
189
        UInt8 type;
190
        readBinary(type, buf);
191
        x.push_back(getBinaryValue(type, buf));
192
    }
193
}
194

195
void writeBinary(const Tuple & x, WriteBuffer & buf)
196
{
197
    const size_t size = x.size();
198
    writeBinary(size, buf);
199

200
    for (const auto & elem : x)
201
    {
202
        const UInt8 type = elem.getType();
203
        writeBinary(type, buf);
204
        Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem);
205
    }
206
}
207

208
void writeText(const Tuple & x, WriteBuffer & buf)
209
{
210
    writeFieldText(Field(x), buf);
211
}
212

213
void readBinary(Map & x, ReadBuffer & buf)
214
{
215
    size_t size;
216
    readBinary(size, buf);
217

218
    for (size_t index = 0; index < size; ++index)
219
    {
220
        UInt8 type;
221
        readBinary(type, buf);
222
        x.push_back(getBinaryValue(type, buf));
223
    }
224
}
225

226
void writeBinary(const Map & x, WriteBuffer & buf)
227
{
228
    const size_t size = x.size();
229
    writeBinary(size, buf);
230

231
    for (const auto & elem : x)
232
    {
233
        const UInt8 type = elem.getType();
234
        writeBinary(type, buf);
235
        Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem);
236
    }
237
}
238

239
void writeText(const Map & x, WriteBuffer & buf)
240
{
241
    writeFieldText(Field(x), buf);
242
}
243

244
void readBinary(Object & x, ReadBuffer & buf)
245
{
246
    size_t size;
247
    readBinary(size, buf);
248

249
    for (size_t index = 0; index < size; ++index)
250
    {
251
        UInt8 type;
252
        String key;
253
        readBinary(type, buf);
254
        readBinary(key, buf);
255
        x[key] = getBinaryValue(type, buf);
256
    }
257
}
258

259
void writeBinary(const Object & x, WriteBuffer & buf)
260
{
261
    const size_t size = x.size();
262
    writeBinary(size, buf);
263

264
    for (const auto & [key, value] : x)
265
    {
266
        const UInt8 type = value.getType();
267
        writeBinary(type, buf);
268
        writeBinary(key, buf);
269
        Field::dispatch([&buf] (const auto & val) { FieldVisitorWriteBinary()(val, buf); }, value);
270
    }
271
}
272

273
void writeText(const Object & x, WriteBuffer & buf)
274
{
275
    writeFieldText(Field(x), buf);
276
}
277

278
void writeBinary(const CustomType & x, WriteBuffer & buf)
279
{
280
    writeBinary(std::string_view(x.getTypeName()), buf);
281
    writeBinary(x.toString(), buf);
282
}
283

284
void writeText(const CustomType & x, WriteBuffer & buf)
285
{
286
    writeFieldText(Field(x), buf);
287
}
288

289
template <typename T>
290
void readQuoted(DecimalField<T> & x, ReadBuffer & buf)
291
{
292
    assertChar('\'', buf);
293
    T value;
294
    UInt32 scale;
295
    int32_t exponent;
296
    uint32_t max_digits = static_cast<uint32_t>(-1);
297
    readDigits<true>(buf, value, max_digits, exponent, true);
298
    if (exponent > 0)
299
    {
300
        scale = 0;
301
        if (common::mulOverflow(value.value, DecimalUtils::scaleMultiplier<T>(exponent), value.value))
302
            throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal math overflow");
303
    }
304
    else
305
        scale = -exponent;
306
    assertChar('\'', buf);
307
    x = DecimalField<T>{value, scale};
308
}
309

310
template void readQuoted<Decimal32>(DecimalField<Decimal32> & x, ReadBuffer & buf);
311
template void readQuoted<Decimal64>(DecimalField<Decimal64> & x, ReadBuffer & buf);
312
template void readQuoted<Decimal128>(DecimalField<Decimal128> & x, ReadBuffer & buf);
313
template void readQuoted<Decimal256>(DecimalField<Decimal256> & x, ReadBuffer & buf);
314

315
void writeFieldText(const Field & x, WriteBuffer & buf)
316
{
317
    String res = Field::dispatch(FieldVisitorToString(), x);
318
    buf.write(res.data(), res.size());
319
}
320

321

322
String Field::dump() const
323
{
324
    return applyVisitor(FieldVisitorDump(), *this);
325
}
326

327
Field Field::restoreFromDump(std::string_view dump_)
328
{
329
    auto show_error = [&dump_]
330
    {
331
        throw Exception(ErrorCodes::CANNOT_RESTORE_FROM_FIELD_DUMP, "Couldn't restore Field from dump: {}", String{dump_});
332
    };
333

334
    std::string_view dump = dump_;
335
    trim(dump);
336

337
    if (dump == "NULL")
338
        return {};
339

340
    std::string_view prefix = std::string_view{"Int64_"};
341
    if (dump.starts_with(prefix))
342
    {
343
        Int64 value = parseFromString<Int64>(dump.substr(prefix.length()));
344
        return value;
345
    }
346

347
    prefix = std::string_view{"UInt64_"};
348
    if (dump.starts_with(prefix))
349
    {
350
        UInt64 value = parseFromString<UInt64>(dump.substr(prefix.length()));
351
        return value;
352
    }
353

354
    prefix = std::string_view{"Int128_"};
355
    if (dump.starts_with(prefix))
356
    {
357
        Int128 value = parseFromString<Int128>(dump.substr(prefix.length()));
358
        return value;
359
    }
360

361
    prefix = std::string_view{"UInt128_"};
362
    if (dump.starts_with(prefix))
363
    {
364
        UInt128 value = parseFromString<UInt128>(dump.substr(prefix.length()));
365
        return value;
366
    }
367

368
    prefix = std::string_view{"Int256_"};
369
    if (dump.starts_with(prefix))
370
    {
371
        Int256 value = parseFromString<Int256>(dump.substr(prefix.length()));
372
        return value;
373
    }
374

375
    prefix = std::string_view{"UInt256_"};
376
    if (dump.starts_with(prefix))
377
    {
378
        UInt256 value = parseFromString<UInt256>(dump.substr(prefix.length()));
379
        return value;
380
    }
381

382
    prefix = std::string_view{"Float64_"};
383
    if (dump.starts_with(prefix))
384
    {
385
        Float64 value = parseFromString<Float64>(dump.substr(prefix.length()));
386
        return value;
387
    }
388

389
    prefix = std::string_view{"Decimal32_"};
390
    if (dump_.starts_with(prefix))
391
    {
392
        DecimalField<Decimal32> decimal;
393
        ReadBufferFromString buf{dump.substr(prefix.length())};
394
        readQuoted(decimal, buf);
395
        return decimal;
396
    }
397

398
    prefix = std::string_view{"Decimal64_"};
399
    if (dump_.starts_with(prefix))
400
    {
401
        DecimalField<Decimal64> decimal;
402
        ReadBufferFromString buf{dump.substr(prefix.length())};
403
        readQuoted(decimal, buf);
404
        return decimal;
405
    }
406

407
    prefix = std::string_view{"Decimal128_"};
408
    if (dump_.starts_with(prefix))
409
    {
410
        DecimalField<Decimal128> decimal;
411
        ReadBufferFromString buf{dump.substr(prefix.length())};
412
        readQuoted(decimal, buf);
413
        return decimal;
414
    }
415

416
    prefix = std::string_view{"Decimal256_"};
417
    if (dump_.starts_with(prefix))
418
    {
419
        DecimalField<Decimal256> decimal;
420
        ReadBufferFromString buf{dump.substr(prefix.length())};
421
        readQuoted(decimal, buf);
422
        return decimal;
423
    }
424

425
    if (dump.starts_with("\'"))
426
    {
427
        String str;
428
        ReadBufferFromString buf{dump};
429
        readQuoted(str, buf);
430
        return str;
431
    }
432

433
    prefix = std::string_view{"Bool_"};
434
    if (dump.starts_with(prefix))
435
    {
436
        bool value = parseFromString<bool>(dump.substr(prefix.length()));
437
        return value;
438
    }
439

440
    prefix = std::string_view{"Array_["};
441
    if (dump.starts_with(prefix))
442
    {
443
        std::string_view tail = dump.substr(prefix.length());
444
        trimLeft(tail);
445
        Array array;
446
        while (tail != "]")
447
        {
448
            size_t separator = tail.find_first_of(",]");
449
            if (separator == std::string_view::npos)
450
                show_error();
451
            bool comma = (tail[separator] == ',');
452
            std::string_view element = tail.substr(0, separator);
453
            tail.remove_prefix(separator);
454
            if (comma)
455
                tail.remove_prefix(1);
456
            trimLeft(tail);
457
            if (!comma && tail != "]")
458
                show_error();
459
            array.push_back(Field::restoreFromDump(element));
460
        }
461
        return array;
462
    }
463

464
    prefix = std::string_view{"Tuple_("};
465
    if (dump.starts_with(prefix))
466
    {
467
        std::string_view tail = dump.substr(prefix.length());
468
        trimLeft(tail);
469
        Tuple tuple;
470
        while (tail != ")")
471
        {
472
            size_t separator = tail.find_first_of(",)");
473
            if (separator == std::string_view::npos)
474
                show_error();
475
            bool comma = (tail[separator] == ',');
476
            std::string_view element = tail.substr(0, separator);
477
            tail.remove_prefix(separator);
478
            if (comma)
479
                tail.remove_prefix(1);
480
            trimLeft(tail);
481
            if (!comma && tail != ")")
482
                show_error();
483
            tuple.push_back(Field::restoreFromDump(element));
484
        }
485
        return tuple;
486
    }
487

488
    prefix = std::string_view{"Map_("};
489
    if (dump.starts_with(prefix))
490
    {
491
        std::string_view tail = dump.substr(prefix.length());
492
        trimLeft(tail);
493
        Map map;
494
        while (tail != ")")
495
        {
496
            size_t separator = tail.find_first_of(",)");
497
            if (separator == std::string_view::npos)
498
                show_error();
499
            bool comma = (tail[separator] == ',');
500
            std::string_view element = tail.substr(0, separator);
501
            tail.remove_prefix(separator);
502
            if (comma)
503
                tail.remove_prefix(1);
504
            trimLeft(tail);
505
            if (!comma && tail != ")")
506
                show_error();
507
            map.push_back(Field::restoreFromDump(element));
508
        }
509
        return map;
510
    }
511

512
    prefix = std::string_view{"AggregateFunctionState_("};
513
    if (dump.starts_with(prefix))
514
    {
515
        std::string_view after_prefix = dump.substr(prefix.length());
516
        size_t comma = after_prefix.find(',');
517
        size_t end = after_prefix.find(')', comma + 1);
518
        if ((comma == std::string_view::npos) || (end != after_prefix.length() - 1))
519
            show_error();
520
        std::string_view name_view = after_prefix.substr(0, comma);
521
        std::string_view data_view = after_prefix.substr(comma + 1, end - comma - 1);
522
        trim(name_view);
523
        trim(data_view);
524
        ReadBufferFromString name_buf{name_view};
525
        ReadBufferFromString data_buf{data_view};
526
        AggregateFunctionStateData res;
527
        readQuotedString(res.name, name_buf);
528
        readQuotedString(res.data, data_buf);
529
        return res;
530
    }
531

532
    show_error();
533
    UNREACHABLE();
534
}
535

536

537
template <typename T>
538
bool decimalEqual(T x, T y, UInt32 x_scale, UInt32 y_scale)
539
{
540
    using Comparator = DecimalComparison<T, T, EqualsOp>;
541
    return Comparator::compare(x, y, x_scale, y_scale);
542
}
543

544
template <typename T>
545
bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale)
546
{
547
    using Comparator = DecimalComparison<T, T, LessOp>;
548
    return Comparator::compare(x, y, x_scale, y_scale);
549
}
550

551
template <typename T>
552
bool decimalLessOrEqual(T x, T y, UInt32 x_scale, UInt32 y_scale)
553
{
554
    using Comparator = DecimalComparison<T, T, LessOrEqualsOp>;
555
    return Comparator::compare(x, y, x_scale, y_scale);
556
}
557

558

559
template bool decimalEqual<Decimal32>(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale);
560
template bool decimalEqual<Decimal64>(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale);
561
template bool decimalEqual<Decimal128>(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale);
562
template bool decimalEqual<Decimal256>(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale);
563
template bool decimalEqual<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
564

565
template bool decimalLess<Decimal32>(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale);
566
template bool decimalLess<Decimal64>(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale);
567
template bool decimalLess<Decimal128>(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale);
568
template bool decimalLess<Decimal256>(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale);
569
template bool decimalLess<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
570

571
template bool decimalLessOrEqual<Decimal32>(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale);
572
template bool decimalLessOrEqual<Decimal64>(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale);
573
template bool decimalLessOrEqual<Decimal128>(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale);
574
template bool decimalLessOrEqual<Decimal256>(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale);
575
template bool decimalLessOrEqual<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
576

577

578
inline void writeText(const Null & x, WriteBuffer & buf)
579
{
580
    if (x.isNegativeInfinity())
581
        writeText("-Inf", buf);
582
    if (x.isPositiveInfinity())
583
        writeText("+Inf", buf);
584
    else
585
        writeText("NULL", buf);
586
}
587

588
String toString(const Field & x)
589
{
590
    return Field::dispatch(
591
        [] (const auto & value)
592
        {
593
            // Use explicit type to prevent implicit construction of Field and
594
            // infinite recursion into toString<Field>.
595
            return toString<decltype(value)>(value);
596
        },
597
        x);
598
}
599

600
std::string_view fieldTypeToString(Field::Types::Which type)
601
{
602
    switch (type)
603
    {
604
        case Field::Types::Which::Null: return "Null"sv;
605
        case Field::Types::Which::Array: return "Array"sv;
606
        case Field::Types::Which::Tuple: return "Tuple"sv;
607
        case Field::Types::Which::Map: return "Map"sv;
608
        case Field::Types::Which::Object: return "Object"sv;
609
        case Field::Types::Which::AggregateFunctionState: return "AggregateFunctionState"sv;
610
        case Field::Types::Which::Bool: return "Bool"sv;
611
        case Field::Types::Which::String: return "String"sv;
612
        case Field::Types::Which::Decimal32: return "Decimal32"sv;
613
        case Field::Types::Which::Decimal64: return "Decimal64"sv;
614
        case Field::Types::Which::Decimal128: return "Decimal128"sv;
615
        case Field::Types::Which::Decimal256: return "Decimal256"sv;
616
        case Field::Types::Which::Float64: return "Float64"sv;
617
        case Field::Types::Which::Int64: return "Int64"sv;
618
        case Field::Types::Which::Int128: return "Int128"sv;
619
        case Field::Types::Which::Int256: return "Int256"sv;
620
        case Field::Types::Which::UInt64: return "UInt64"sv;
621
        case Field::Types::Which::UInt128: return "UInt128"sv;
622
        case Field::Types::Which::UInt256: return "UInt256"sv;
623
        case Field::Types::Which::UUID: return "UUID"sv;
624
        case Field::Types::Which::IPv4: return "IPv4"sv;
625
        case Field::Types::Which::IPv6: return "IPv6"sv;
626
        case Field::Types::Which::CustomType: return "CustomType"sv;
627
    }
628
}
629

630
/// Keep in mind, that "magic_enum" is very expensive for compiler, that's why we don't use it.
631
std::string_view Field::getTypeName() const
632
{
633
    return fieldTypeToString(which);
634
}
635

636
template class DecimalField<Decimal32>;
637
template class DecimalField<Decimal64>;
638
template class DecimalField<Decimal128>;
639
template class DecimalField<Decimal256>;
640
template class DecimalField<DateTime64>;
641
}
642