ClickHouse

1
#include <Processors/Chunk.h>
2
#include <IO/WriteHelpers.h>
3
#include <IO/Operators.h>
4
#include <Columns/ColumnSparse.h>
5
#include <Columns/ColumnConst.h>
6
#include <DataTypes/DataTypeLowCardinality.h>
7

8
namespace DB
9
{
10

11
namespace ErrorCodes
12
{
13
    extern const int LOGICAL_ERROR;
14
    extern const int POSITION_OUT_OF_BOUND;
15
}
16

17
Chunk::Chunk(DB::Columns columns_, UInt64 num_rows_) : columns(std::move(columns_)), num_rows(num_rows_)
18
{
19
    checkNumRowsIsConsistent();
20
}
21

22
Chunk::Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_)
23
    : columns(std::move(columns_))
24
    , num_rows(num_rows_)
25
    , chunk_info(std::move(chunk_info_))
26
{
27
    checkNumRowsIsConsistent();
28
}
29

30
static Columns unmuteColumns(MutableColumns && mutable_columns)
31
{
32
    Columns columns;
33
    columns.reserve(mutable_columns.size());
34
    for (auto & col : mutable_columns)
35
        columns.emplace_back(std::move(col));
36

37
    return columns;
38
}
39

40
Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_)
41
    : columns(unmuteColumns(std::move(columns_))), num_rows(num_rows_)
42
{
43
    checkNumRowsIsConsistent();
44
}
45

46
Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_)
47
    : columns(unmuteColumns(std::move(columns_)))
48
    , num_rows(num_rows_)
49
    , chunk_info(std::move(chunk_info_))
50
{
51
    checkNumRowsIsConsistent();
52
}
53

54
Chunk Chunk::clone() const
55
{
56
    return Chunk(getColumns(), getNumRows(), chunk_info);
57
}
58

59
void Chunk::setColumns(Columns columns_, UInt64 num_rows_)
60
{
61
    columns = std::move(columns_);
62
    num_rows = num_rows_;
63
    checkNumRowsIsConsistent();
64
}
65

66
void Chunk::setColumns(MutableColumns columns_, UInt64 num_rows_)
67
{
68
    columns = unmuteColumns(std::move(columns_));
69
    num_rows = num_rows_;
70
    checkNumRowsIsConsistent();
71
}
72

73
void Chunk::checkNumRowsIsConsistent()
74
{
75
    for (size_t i = 0; i < columns.size(); ++i)
76
    {
77
        auto & column = columns[i];
78
        if (column->size() != num_rows)
79
            throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid number of rows in Chunk column {}: expected {}, got {}",
80
                            column->getName() + " position " + toString(i), toString(num_rows), toString(column->size()));
81
    }
82
}
83

84
MutableColumns Chunk::mutateColumns()
85
{
86
    size_t num_columns = columns.size();
87
    MutableColumns mutable_columns(num_columns);
88
    for (size_t i = 0; i < num_columns; ++i)
89
        mutable_columns[i] = IColumn::mutate(std::move(columns[i]));
90

91
    columns.clear();
92
    num_rows = 0;
93

94
    return mutable_columns;
95
}
96

97
MutableColumns Chunk::cloneEmptyColumns() const
98
{
99
    size_t num_columns = columns.size();
100
    MutableColumns mutable_columns(num_columns);
101
    for (size_t i = 0; i < num_columns; ++i)
102
        mutable_columns[i] = columns[i]->cloneEmpty();
103
    return mutable_columns;
104
}
105

106
Columns Chunk::detachColumns()
107
{
108
    num_rows = 0;
109
    return std::move(columns);
110
}
111

112
void Chunk::addColumn(ColumnPtr column)
113
{
114
    if (empty())
115
        num_rows = column->size();
116
    else if (column->size() != num_rows)
117
        throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid number of rows in Chunk column {}, got {}",
118
                        column->getName()+ ": expected " + toString(num_rows), toString(column->size()));
119

120
    columns.emplace_back(std::move(column));
121
}
122

123
void Chunk::addColumn(size_t position, ColumnPtr column)
124
{
125
    if (position >= columns.size())
126
        throw Exception(ErrorCodes::POSITION_OUT_OF_BOUND,
127
                        "Position {} out of bound in Chunk::addColumn(), max position = {}",
128
                        position, columns.size() - 1);
129
    if (empty())
130
        num_rows = column->size();
131
    else if (column->size() != num_rows)
132
        throw Exception(ErrorCodes::LOGICAL_ERROR,
133
                        "Invalid number of rows in Chunk column {}: expected {}, got {}",
134
                        column->getName(), num_rows, column->size());
135

136
    columns.emplace(columns.begin() + position, std::move(column));
137
}
138

139
void Chunk::erase(size_t position)
140
{
141
    if (columns.empty())
142
        throw Exception(ErrorCodes::POSITION_OUT_OF_BOUND, "Chunk is empty");
143

144
    if (position >= columns.size())
145
        throw Exception(ErrorCodes::POSITION_OUT_OF_BOUND, "Position {} out of bound in Chunk::erase(), max position = {}",
146
                        toString(position), toString(columns.size() - 1));
147

148
    columns.erase(columns.begin() + position);
149
}
150

151
UInt64 Chunk::bytes() const
152
{
153
    UInt64 res = 0;
154
    for (const auto & column : columns)
155
        res += column->byteSize();
156

157
    return res;
158
}
159

160
UInt64 Chunk::allocatedBytes() const
161
{
162
    UInt64 res = 0;
163
    for (const auto & column : columns)
164
        res += column->allocatedBytes();
165

166
    return res;
167
}
168

169
std::string Chunk::dumpStructure() const
170
{
171
    WriteBufferFromOwnString out;
172
    for (const auto & column : columns)
173
        out << ' ' << column->dumpStructure();
174

175
    return out.str();
176
}
177

178
void Chunk::append(const Chunk & chunk)
179
{
180
    append(chunk, 0, chunk.getNumRows());
181
}
182

183
void Chunk::append(const Chunk & chunk, size_t from, size_t length)
184
{
185
    MutableColumns mutable_columns = mutateColumns();
186
    for (size_t position = 0; position < mutable_columns.size(); ++position)
187
    {
188
        auto column = chunk.getColumns()[position];
189
        mutable_columns[position]->insertRangeFrom(*column, from, length);
190
    }
191
    size_t rows = mutable_columns[0]->size();
192
    setColumns(std::move(mutable_columns), rows);
193
}
194

195
void ChunkMissingValues::setBit(size_t column_idx, size_t row_idx)
196
{
197
    RowsBitMask & mask = rows_mask_by_column_id[column_idx];
198
    mask.resize(row_idx + 1);
199
    mask[row_idx] = true;
200
}
201

202
const ChunkMissingValues::RowsBitMask & ChunkMissingValues::getDefaultsBitmask(size_t column_idx) const
203
{
204
    static RowsBitMask none;
205
    auto it = rows_mask_by_column_id.find(column_idx);
206
    if (it != rows_mask_by_column_id.end())
207
        return it->second;
208
    return none;
209
}
210

211
void convertToFullIfConst(Chunk & chunk)
212
{
213
    size_t num_rows = chunk.getNumRows();
214
    auto columns = chunk.detachColumns();
215
    for (auto & column : columns)
216
        column = column->convertToFullColumnIfConst();
217
    chunk.setColumns(std::move(columns), num_rows);
218
}
219

220
void convertToFullIfSparse(Chunk & chunk)
221
{
222
    size_t num_rows = chunk.getNumRows();
223
    auto columns = chunk.detachColumns();
224
    for (auto & column : columns)
225
        column = recursiveRemoveSparse(column);
226
    chunk.setColumns(std::move(columns), num_rows);
227
}
228

229
Chunk cloneConstWithDefault(const Chunk & chunk, size_t num_rows)
230
{
231
    auto columns = chunk.cloneEmptyColumns();
232
    for (auto & column : columns)
233
    {
234
        column->insertDefault();
235
        column = ColumnConst::create(std::move(column), num_rows);
236
    }
237

238
    return Chunk(std::move(columns), num_rows);
239
}
240

241
}
242