ClickHouse

1
#include <Planner/PlannerActionsVisitor.h>
2

3
#include <Analyzer/Utils.h>
4
#include <Analyzer/SetUtils.h>
5
#include <Analyzer/ConstantNode.h>
6
#include <Analyzer/FunctionNode.h>
7
#include <Analyzer/ColumnNode.h>
8
#include <Analyzer/LambdaNode.h>
9
#include <Analyzer/SortNode.h>
10
#include <Analyzer/WindowNode.h>
11
#include <Analyzer/QueryNode.h>
12

13
#include <DataTypes/FieldToDataType.h>
14
#include <DataTypes/DataTypeSet.h>
15

16
#include <Common/FieldVisitorToString.h>
17
#include <DataTypes/DataTypeTuple.h>
18

19
#include <Columns/ColumnSet.h>
20
#include <Columns/ColumnConst.h>
21

22
#include <Functions/FunctionsMiscellaneous.h>
23
#include <Functions/FunctionFactory.h>
24
#include <Functions/indexHint.h>
25

26
#include <Interpreters/ExpressionActions.h>
27
#include <Interpreters/Context.h>
28

29
#include <Planner/PlannerContext.h>
30
#include <Planner/TableExpressionData.h>
31
#include <Planner/Utils.h>
32

33

34
namespace DB
35
{
36

37
namespace ErrorCodes
38
{
39
    extern const int UNSUPPORTED_METHOD;
40
    extern const int LOGICAL_ERROR;
41
    extern const int BAD_ARGUMENTS;
42
    extern const int INCORRECT_QUERY;
43
}
44

45
namespace
46
{
47

48
/* Calculates Action node name for ConstantNode.
49
 *
50
 * If converting to AST will add a '_CAST' function call,
51
 * the result action name will also include it.
52
 */
53
String calculateActionNodeNameWithCastIfNeeded(const ConstantNode & constant_node)
54
{
55
    WriteBufferFromOwnString buffer;
56
    if (constant_node.requiresCastCall())
57
        buffer << "_CAST(";
58

59
    buffer << calculateConstantActionNodeName(constant_node.getValue(), constant_node.getResultType());
60

61
    if (constant_node.requiresCastCall())
62
    {
63
        buffer << ", '" << constant_node.getResultType()->getName() << "'_String)";
64
    }
65

66
    return buffer.str();
67
}
68

69
class ActionNodeNameHelper
70
{
71
public:
72
    ActionNodeNameHelper(QueryTreeNodeToName & node_to_name_,
73
        const PlannerContext & planner_context_,
74
        bool use_column_identifier_as_action_node_name_)
75
        : node_to_name(node_to_name_)
76
        , planner_context(planner_context_)
77
        , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
78
    {
79
    }
80

81
    String calculateActionNodeName(const QueryTreeNodePtr & node)
82
    {
83
        auto it = node_to_name.find(node);
84
        if (it != node_to_name.end())
85
            return it->second;
86

87
        String result;
88
        auto node_type = node->getNodeType();
89

90
        switch (node_type)
91
        {
92
            case QueryTreeNodeType::COLUMN:
93
            {
94
                const ColumnIdentifier * column_identifier = nullptr;
95
                if (use_column_identifier_as_action_node_name)
96
                    column_identifier = planner_context.getColumnNodeIdentifierOrNull(node);
97

98
                if (column_identifier)
99
                {
100
                    result = *column_identifier;
101
                }
102
                else
103
                {
104
                    const auto & column_node = node->as<ColumnNode &>();
105
                    result = column_node.getColumnName();
106
                }
107

108
                break;
109
            }
110
            case QueryTreeNodeType::CONSTANT:
111
            {
112
                const auto & constant_node = node->as<ConstantNode &>();
113
                /* To ensure that headers match during distributed query we need to simulate action node naming on
114
                * secondary servers. If we don't do that headers will mismatch due to constant folding.
115
                *
116
                *                                +--------+
117
                *               -----------------| Server |----------------
118
                *              /                 +--------+                \
119
                *             /                                             \
120
                *            v                                               v
121
                *      +-----------+                                   +-----------+
122
                *      | Initiator |                            ------ | Secondary |------
123
                *      +-----------+                           /       +-----------+      \
124
                *            |                                /                            \
125
                *            |                               /                              \
126
                *            v                              /                                \
127
                *    +---------------+                     v                                  v
128
                *    | Wrap in _CAST |      +----------------------------+        +----------------------+
129
                *    | if needed     |      | Constant folded from _CAST |        | Constant folded from |
130
                *    +---------------+      +----------------------------+        | another expression   |
131
                *                                          |                      +----------------------+
132
                *                                          v                                  |
133
                *                           +----------------------------+                    v
134
                *                           | Name ConstantNode the same |      +--------------------------+
135
                *                           | as on initiator server     |      | Generate action name for |
136
                *                           | (wrap in _CAST if needed)  |      | original expression      |
137
                *                           +----------------------------+      +--------------------------+
138
                */
139
                if (planner_context.isASTLevelOptimizationAllowed())
140
                {
141
                    result = calculateActionNodeNameWithCastIfNeeded(constant_node);
142
                }
143
                else
144
                {
145
                    // Need to check if constant folded from QueryNode until https://github.com/ClickHouse/ClickHouse/issues/60847 is fixed.
146
                    if (constant_node.hasSourceExpression() && constant_node.getSourceExpression()->getNodeType() != QueryTreeNodeType::QUERY)
147
                    {
148
                        if (constant_node.receivedFromInitiatorServer())
149
                            result = calculateActionNodeNameWithCastIfNeeded(constant_node);
150
                        else
151
                            result = calculateActionNodeName(constant_node.getSourceExpression());
152
                    }
153
                    else
154
                        result = calculateConstantActionNodeName(constant_node.getValue(), constant_node.getResultType());
155
                }
156
                break;
157
            }
158
            case QueryTreeNodeType::FUNCTION:
159
            {
160
                const auto & function_node = node->as<FunctionNode &>();
161
                if (function_node.getFunctionName() == "__actionName")
162
                {
163
                    result = toString(function_node.getArguments().getNodes().at(1)->as<ConstantNode>()->getValue());
164
                    break;
165
                }
166

167
                String in_function_second_argument_node_name;
168

169
                if (isNameOfInFunction(function_node.getFunctionName()))
170
                {
171
                    const auto & in_first_argument_node = function_node.getArguments().getNodes().at(0);
172
                    const auto & in_second_argument_node = function_node.getArguments().getNodes().at(1);
173
                    in_function_second_argument_node_name = PlannerContext::createSetKey(in_first_argument_node->getResultType(), in_second_argument_node);
174
                }
175

176
                WriteBufferFromOwnString buffer;
177
                buffer << function_node.getFunctionName();
178

179
                const auto & function_parameters_nodes = function_node.getParameters().getNodes();
180

181
                if (!function_parameters_nodes.empty())
182
                {
183
                    buffer << '(';
184

185
                    size_t function_parameters_nodes_size = function_parameters_nodes.size();
186
                    for (size_t i = 0; i < function_parameters_nodes_size; ++i)
187
                    {
188
                        const auto & function_parameter_node = function_parameters_nodes[i];
189
                        buffer << calculateActionNodeName(function_parameter_node);
190

191
                        if (i + 1 != function_parameters_nodes_size)
192
                            buffer << ", ";
193
                    }
194

195
                    buffer << ')';
196
                }
197

198
                const auto & function_arguments_nodes = function_node.getArguments().getNodes();
199
                String function_argument_name;
200

201
                buffer << '(';
202

203
                size_t function_arguments_nodes_size = function_arguments_nodes.size();
204
                for (size_t i = 0; i < function_arguments_nodes_size; ++i)
205
                {
206
                    if (i == 1 && !in_function_second_argument_node_name.empty())
207
                    {
208
                        function_argument_name = in_function_second_argument_node_name;
209
                    }
210
                    else
211
                    {
212
                        const auto & function_argument_node = function_arguments_nodes[i];
213
                        function_argument_name = calculateActionNodeName(function_argument_node);
214
                    }
215

216
                    buffer << function_argument_name;
217

218
                    if (i + 1 != function_arguments_nodes_size)
219
                        buffer << ", ";
220
                }
221

222
                buffer << ')';
223

224
                if (function_node.isWindowFunction())
225
                {
226
                    buffer << " OVER (";
227
                    buffer << calculateWindowNodeActionName(function_node.getWindowNode());
228
                    buffer << ')';
229
                }
230

231
                result = buffer.str();
232
                break;
233
            }
234
            case QueryTreeNodeType::LAMBDA:
235
            {
236
                auto lambda_hash = node->getTreeHash();
237
                result = "__lambda_" + toString(lambda_hash);
238
                break;
239
            }
240
            default:
241
            {
242
                throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid action query tree node {}", node->formatASTForErrorMessage());
243
            }
244
        }
245

246
        node_to_name.emplace(node, result);
247

248
        return result;
249
    }
250

251
    static String calculateConstantActionNodeName(const Field & constant_literal, const DataTypePtr & constant_type)
252
    {
253
        auto constant_name = applyVisitor(FieldVisitorToString(), constant_literal);
254
        return constant_name + "_" + constant_type->getName();
255
    }
256

257
    static String calculateConstantActionNodeName(const Field & constant_literal)
258
    {
259
        return calculateConstantActionNodeName(constant_literal, applyVisitor(FieldToDataType(), constant_literal));
260
    }
261

262
    String calculateWindowNodeActionName(const QueryTreeNodePtr & node)
263
    {
264
        auto & window_node = node->as<WindowNode &>();
265
        WriteBufferFromOwnString buffer;
266

267
        if (window_node.hasPartitionBy())
268
        {
269
            buffer << "PARTITION BY ";
270

271
            auto & partition_by_nodes = window_node.getPartitionBy().getNodes();
272
            size_t partition_by_nodes_size = partition_by_nodes.size();
273

274
            for (size_t i = 0; i < partition_by_nodes_size; ++i)
275
            {
276
                auto & partition_by_node = partition_by_nodes[i];
277
                buffer << calculateActionNodeName(partition_by_node);
278
                if (i + 1 != partition_by_nodes_size)
279
                    buffer << ", ";
280
            }
281
        }
282

283
        if (window_node.hasOrderBy())
284
        {
285
            if (window_node.hasPartitionBy())
286
                buffer << ' ';
287

288
            buffer << "ORDER BY ";
289

290
            auto & order_by_nodes = window_node.getOrderBy().getNodes();
291
            size_t order_by_nodes_size = order_by_nodes.size();
292

293
            for (size_t i = 0; i < order_by_nodes_size; ++i)
294
            {
295
                auto & sort_node = order_by_nodes[i]->as<SortNode &>();
296
                buffer << calculateActionNodeName(sort_node.getExpression());
297

298
                auto sort_direction = sort_node.getSortDirection();
299
                buffer << (sort_direction == SortDirection::ASCENDING ? " ASC" : " DESC");
300

301
                auto nulls_sort_direction = sort_node.getNullsSortDirection();
302

303
                if (nulls_sort_direction)
304
                    buffer << " NULLS " << (nulls_sort_direction == sort_direction ? "LAST" : "FIRST");
305

306
                if (auto collator = sort_node.getCollator())
307
                    buffer << " COLLATE " << collator->getLocale();
308

309
                if (sort_node.withFill())
310
                {
311
                    buffer << " WITH FILL";
312

313
                    if (sort_node.hasFillFrom())
314
                        buffer << " FROM " << calculateActionNodeName(sort_node.getFillFrom());
315

316
                    if (sort_node.hasFillTo())
317
                        buffer << " TO " << calculateActionNodeName(sort_node.getFillTo());
318

319
                    if (sort_node.hasFillStep())
320
                        buffer << " STEP " << calculateActionNodeName(sort_node.getFillStep());
321
                }
322

323
                if (i + 1 != order_by_nodes_size)
324
                    buffer << ", ";
325
            }
326
        }
327

328
        auto & window_frame = window_node.getWindowFrame();
329
        if (!window_frame.is_default)
330
        {
331
            if (window_node.hasPartitionBy() || window_node.hasOrderBy())
332
                buffer << ' ';
333

334
            buffer << window_frame.type << " BETWEEN ";
335
            if (window_frame.begin_type == WindowFrame::BoundaryType::Current)
336
            {
337
                buffer << "CURRENT ROW";
338
            }
339
            else if (window_frame.begin_type == WindowFrame::BoundaryType::Unbounded)
340
            {
341
                buffer << "UNBOUNDED";
342
                buffer << " " << (window_frame.begin_preceding ? "PRECEDING" : "FOLLOWING");
343
            }
344
            else
345
            {
346
                buffer << calculateActionNodeName(window_node.getFrameBeginOffsetNode());
347
                buffer << " " << (window_frame.begin_preceding ? "PRECEDING" : "FOLLOWING");
348
            }
349

350
            buffer << " AND ";
351

352
            if (window_frame.end_type == WindowFrame::BoundaryType::Current)
353
            {
354
                buffer << "CURRENT ROW";
355
            }
356
            else if (window_frame.end_type == WindowFrame::BoundaryType::Unbounded)
357
            {
358
                buffer << "UNBOUNDED";
359
                buffer << " " << (window_frame.end_preceding ? "PRECEDING" : "FOLLOWING");
360
            }
361
            else
362
            {
363
                buffer << calculateActionNodeName(window_node.getFrameEndOffsetNode());
364
                buffer << " " << (window_frame.end_preceding ? "PRECEDING" : "FOLLOWING");
365
            }
366
        }
367

368
        return buffer.str();
369
    }
370
private:
371
    std::unordered_map<QueryTreeNodePtr, std::string> & node_to_name;
372
    const PlannerContext & planner_context;
373
    bool use_column_identifier_as_action_node_name = true;
374
};
375

376
class ActionsScopeNode
377
{
378
public:
379
    explicit ActionsScopeNode(ActionsDAGPtr actions_dag_, QueryTreeNodePtr scope_node_)
380
        : actions_dag(std::move(actions_dag_))
381
        , scope_node(std::move(scope_node_))
382
    {
383
        for (const auto & node : actions_dag->getNodes())
384
            node_name_to_node[node.result_name] = &node;
385
    }
386

387
    const QueryTreeNodePtr & getScopeNode() const
388
    {
389
        return scope_node;
390
    }
391

392
    [[maybe_unused]] bool containsNode(const std::string & node_name)
393
    {
394
        return node_name_to_node.find(node_name) != node_name_to_node.end();
395
    }
396

397
    [[maybe_unused]] bool containsInputNode(const std::string & node_name)
398
    {
399
        const auto * node = tryGetNode(node_name);
400
        if (node && node->type == ActionsDAG::ActionType::INPUT)
401
            return true;
402

403
        return false;
404
    }
405

406
    [[maybe_unused]] const ActionsDAG::Node * tryGetNode(const std::string & node_name)
407
    {
408
        auto it = node_name_to_node.find(node_name);
409
        if (it == node_name_to_node.end())
410
            return {};
411

412
        return it->second;
413
    }
414

415
    const ActionsDAG::Node * getNodeOrThrow(const std::string & node_name)
416
    {
417
        auto it = node_name_to_node.find(node_name);
418
        if (it == node_name_to_node.end())
419
            throw Exception(ErrorCodes::LOGICAL_ERROR,
420
                "No node with name {}. There are only nodes {}",
421
                node_name,
422
                actions_dag->dumpNames());
423

424
        return it->second;
425
    }
426

427
    const ActionsDAG::Node * addInputColumnIfNecessary(const std::string & node_name, const DataTypePtr & column_type)
428
    {
429
        auto it = node_name_to_node.find(node_name);
430
        if (it != node_name_to_node.end())
431
            return it->second;
432

433
        const auto * node = &actions_dag->addInput(node_name, column_type);
434
        node_name_to_node[node->result_name] = node;
435

436
        return node;
437
    }
438

439
    const ActionsDAG::Node * addInputConstantColumnIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column)
440
    {
441
        auto it = node_name_to_node.find(node_name);
442
        if (it != node_name_to_node.end())
443
            return it->second;
444

445
        const auto * node = &actions_dag->addInput(column);
446
        node_name_to_node[node->result_name] = node;
447

448
        return node;
449
    }
450

451
    const ActionsDAG::Node * addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column)
452
    {
453
        auto it = node_name_to_node.find(node_name);
454
        if (it != node_name_to_node.end())
455
            return it->second;
456

457
        const auto * node = &actions_dag->addColumn(column);
458
        node_name_to_node[node->result_name] = node;
459

460
        return node;
461
    }
462

463
    template <typename FunctionOrOverloadResolver>
464
    const ActionsDAG::Node * addFunctionIfNecessary(const std::string & node_name, ActionsDAG::NodeRawConstPtrs children, const FunctionOrOverloadResolver & function)
465
    {
466
        auto it = node_name_to_node.find(node_name);
467
        if (it != node_name_to_node.end())
468
            return it->second;
469

470
        const auto * node = &actions_dag->addFunction(function, children, node_name);
471
        node_name_to_node[node->result_name] = node;
472

473
        return node;
474
    }
475

476
    const ActionsDAG::Node * addArrayJoinIfNecessary(const std::string & node_name, const ActionsDAG::Node * child)
477
    {
478
        auto it = node_name_to_node.find(node_name);
479
        if (it != node_name_to_node.end())
480
            return it->second;
481

482
        const auto * node = &actions_dag->addArrayJoin(*child, node_name);
483
        node_name_to_node[node->result_name] = node;
484

485
        return node;
486
    }
487

488
private:
489
    std::unordered_map<std::string_view, const ActionsDAG::Node *> node_name_to_node;
490
    ActionsDAGPtr actions_dag;
491
    QueryTreeNodePtr scope_node;
492
};
493

494
class PlannerActionsVisitorImpl
495
{
496
public:
497
    PlannerActionsVisitorImpl(ActionsDAGPtr actions_dag,
498
        const PlannerContextPtr & planner_context_,
499
        bool use_column_identifier_as_action_node_name_);
500

501
    ActionsDAG::NodeRawConstPtrs visit(QueryTreeNodePtr expression_node);
502

503
private:
504

505
    class Levels
506
    {
507
    public:
508
        explicit Levels(size_t level) { set(level); }
509

510
        void set(size_t level)
511
        {
512
            check(level);
513
            if (level)
514
                mask |= (uint64_t(1) << (level - 1));
515
        }
516

517
        void reset(size_t level)
518
        {
519
            check(level);
520
            if (level)
521
                mask &= ~(uint64_t(1) << (level - 1));
522
        }
523

524
        void add(Levels levels) { mask |= levels.mask; }
525

526
        size_t max() const { return 64 - getLeadingZeroBits(mask); }
527

528
    private:
529
        uint64_t mask = 0;
530

531
        void check(size_t level)
532
        {
533
            if (level > 64)
534
                throw Exception(ErrorCodes::INCORRECT_QUERY, "Maximum lambda depth exceeded. Maximum 64.");
535
        }
536
    };
537

538
    using NodeNameAndNodeMinLevel = std::pair<std::string, Levels>;
539

540
    NodeNameAndNodeMinLevel visitImpl(QueryTreeNodePtr node);
541

542
    NodeNameAndNodeMinLevel visitColumn(const QueryTreeNodePtr & node);
543

544
    NodeNameAndNodeMinLevel visitConstant(const QueryTreeNodePtr & node);
545

546
    NodeNameAndNodeMinLevel visitLambda(const QueryTreeNodePtr & node);
547

548
    NodeNameAndNodeMinLevel makeSetForInFunction(const QueryTreeNodePtr & node);
549

550
    NodeNameAndNodeMinLevel visitIndexHintFunction(const QueryTreeNodePtr & node);
551

552
    NodeNameAndNodeMinLevel visitFunction(const QueryTreeNodePtr & node);
553

554
    std::vector<ActionsScopeNode> actions_stack;
555
    std::unordered_map<QueryTreeNodePtr, std::string> node_to_node_name;
556
    const PlannerContextPtr planner_context;
557
    ActionNodeNameHelper action_node_name_helper;
558
    bool use_column_identifier_as_action_node_name;
559
};
560

561
PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAGPtr actions_dag,
562
    const PlannerContextPtr & planner_context_,
563
    bool use_column_identifier_as_action_node_name_)
564
    : planner_context(planner_context_)
565
    , action_node_name_helper(node_to_node_name, *planner_context, use_column_identifier_as_action_node_name_)
566
    , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
567
{
568
    actions_stack.emplace_back(std::move(actions_dag), nullptr);
569
}
570

571
ActionsDAG::NodeRawConstPtrs PlannerActionsVisitorImpl::visit(QueryTreeNodePtr expression_node)
572
{
573
    ActionsDAG::NodeRawConstPtrs result;
574

575
    if (auto * expression_list_node = expression_node->as<ListNode>())
576
    {
577
        for (auto & node : expression_list_node->getNodes())
578
        {
579
            auto [node_name, _] = visitImpl(node);
580
            result.push_back(actions_stack.front().getNodeOrThrow(node_name));
581
        }
582
    }
583
    else
584
    {
585
        auto [node_name, _] = visitImpl(expression_node);
586
        result.push_back(actions_stack.front().getNodeOrThrow(node_name));
587
    }
588

589
    return result;
590
}
591

592
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitImpl(QueryTreeNodePtr node)
593
{
594
    auto node_type = node->getNodeType();
595

596
    if (node_type == QueryTreeNodeType::COLUMN)
597
        return visitColumn(node);
598
    else if (node_type == QueryTreeNodeType::CONSTANT)
599
        return visitConstant(node);
600
    else if (node_type == QueryTreeNodeType::FUNCTION)
601
        return visitFunction(node);
602

603
    throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
604
        "Expected column, constant, function. Actual {} with type: {}",
605
        node->formatASTForErrorMessage(), node_type);
606
}
607

608
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitColumn(const QueryTreeNodePtr & node)
609
{
610
    auto column_node_name = action_node_name_helper.calculateActionNodeName(node);
611
    const auto & column_node = node->as<ColumnNode &>();
612
    if (column_node.hasExpression() && !use_column_identifier_as_action_node_name)
613
        return visitImpl(column_node.getExpression());
614
    Int64 actions_stack_size = static_cast<Int64>(actions_stack.size() - 1);
615
    for (Int64 i = actions_stack_size; i >= 0; --i)
616
    {
617
        actions_stack[i].addInputColumnIfNecessary(column_node_name, column_node.getColumnType());
618

619
        auto column_source = column_node.getColumnSourceOrNull();
620
        if (column_source &&
621
            column_source->getNodeType() == QueryTreeNodeType::LAMBDA &&
622
            actions_stack[i].getScopeNode().get() == column_source.get())
623
        {
624
            return {column_node_name, Levels(i)};
625
        }
626
    }
627

628
    return {column_node_name, Levels(0)};
629
}
630

631
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitConstant(const QueryTreeNodePtr & node)
632
{
633
    const auto & constant_node = node->as<ConstantNode &>();
634
    const auto & constant_literal = constant_node.getValue();
635
    const auto & constant_type = constant_node.getResultType();
636

637
    auto constant_node_name = [&]()
638
    {
639
        /* To ensure that headers match during distributed query we need to simulate action node naming on
640
         * secondary servers. If we don't do that headers will mismatch due to constant folding.
641
         *
642
         *                                +--------+
643
         *               -----------------| Server |----------------
644
         *              /                 +--------+                \
645
         *             /                                             \
646
         *            v                                               v
647
         *      +-----------+                                   +-----------+
648
         *      | Initiator |                            ------ | Secondary |------
649
         *      +-----------+                           /       +-----------+      \
650
         *            |                                /                            \
651
         *            |                               /                              \
652
         *            v                              /                                \
653
         *    +---------------+                     v                                  v
654
         *    | Wrap in _CAST |      +----------------------------+        +----------------------+
655
         *    | if needed     |      | Constant folded from _CAST |        | Constant folded from |
656
         *    +---------------+      +----------------------------+        | another expression   |
657
         *                                          |                      +----------------------+
658
         *                                          v                                  |
659
         *                           +----------------------------+                    v
660
         *                           | Name ConstantNode the same |      +--------------------------+
661
         *                           | as on initiator server     |      | Generate action name for |
662
         *                           | (wrap in _CAST if needed)  |      | original expression      |
663
         *                           +----------------------------+      +--------------------------+
664
         */
665
        if (planner_context->isASTLevelOptimizationAllowed())
666
        {
667
            return calculateActionNodeNameWithCastIfNeeded(constant_node);
668
        }
669
        else
670
        {
671
            // Need to check if constant folded from QueryNode until https://github.com/ClickHouse/ClickHouse/issues/60847 is fixed.
672
            if (constant_node.hasSourceExpression() && constant_node.getSourceExpression()->getNodeType() != QueryTreeNodeType::QUERY)
673
            {
674
                if (constant_node.receivedFromInitiatorServer())
675
                    return calculateActionNodeNameWithCastIfNeeded(constant_node);
676
                else
677
                    return action_node_name_helper.calculateActionNodeName(constant_node.getSourceExpression());
678
            }
679
            else
680
                return calculateConstantActionNodeName(constant_literal, constant_type);
681
        }
682
    }();
683

684
    ColumnWithTypeAndName column;
685
    column.name = constant_node_name;
686
    column.type = constant_type;
687
    column.column = column.type->createColumnConst(1, constant_literal);
688

689
    actions_stack[0].addConstantIfNecessary(constant_node_name, column);
690

691
    size_t actions_stack_size = actions_stack.size();
692
    for (size_t i = 1; i < actions_stack_size; ++i)
693
    {
694
        auto & actions_stack_node = actions_stack[i];
695
        actions_stack_node.addInputConstantColumnIfNecessary(constant_node_name, column);
696
    }
697

698
    return {constant_node_name, Levels(0)};
699

700
}
701

702
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitLambda(const QueryTreeNodePtr & node)
703
{
704
    auto & lambda_node = node->as<LambdaNode &>();
705
    auto result_type = lambda_node.getResultType();
706
    if (!result_type)
707
        throw Exception(ErrorCodes::LOGICAL_ERROR,
708
            "Lambda {} is not resolved during query analysis",
709
            lambda_node.formatASTForErrorMessage());
710

711
    auto & lambda_arguments_nodes = lambda_node.getArguments().getNodes();
712
    size_t lambda_arguments_nodes_size = lambda_arguments_nodes.size();
713

714
    NamesAndTypesList lambda_arguments_names_and_types;
715

716
    for (size_t i = 0; i < lambda_arguments_nodes_size; ++i)
717
    {
718
        const auto & lambda_argument_name = lambda_node.getArgumentNames().at(i);
719
        auto lambda_argument_type = lambda_arguments_nodes[i]->getResultType();
720
        lambda_arguments_names_and_types.emplace_back(lambda_argument_name, std::move(lambda_argument_type));
721
    }
722

723
    auto lambda_actions_dag = std::make_shared<ActionsDAG>();
724
    actions_stack.emplace_back(lambda_actions_dag, node);
725

726
    auto [lambda_expression_node_name, levels] = visitImpl(lambda_node.getExpression());
727
    lambda_actions_dag->getOutputs().push_back(actions_stack.back().getNodeOrThrow(lambda_expression_node_name));
728
    lambda_actions_dag->removeUnusedActions(Names(1, lambda_expression_node_name));
729

730
    auto expression_actions_settings = ExpressionActionsSettings::fromContext(planner_context->getQueryContext(), CompileExpressions::yes);
731
    auto lambda_actions = std::make_shared<ExpressionActions>(lambda_actions_dag, expression_actions_settings);
732

733
    Names captured_column_names;
734
    ActionsDAG::NodeRawConstPtrs lambda_children;
735
    Names required_column_names = lambda_actions->getRequiredColumns();
736

737
    actions_stack.pop_back();
738
    levels.reset(actions_stack.size());
739
    size_t level = levels.max();
740

741
    const auto & lambda_argument_names = lambda_node.getArgumentNames();
742

743
    for (const auto & required_column_name : required_column_names)
744
    {
745
        auto it = std::find(lambda_argument_names.begin(), lambda_argument_names.end(), required_column_name);
746

747
        if (it == lambda_argument_names.end())
748
        {
749
            lambda_children.push_back(actions_stack[level].getNodeOrThrow(required_column_name));
750
            captured_column_names.push_back(required_column_name);
751
        }
752
    }
753

754
    auto lambda_node_name = calculateActionNodeName(node, *planner_context);
755
    auto function_capture = std::make_shared<FunctionCaptureOverloadResolver>(
756
        lambda_actions, captured_column_names, lambda_arguments_names_and_types, lambda_node.getExpression()->getResultType(), lambda_expression_node_name);
757

758
    // TODO: Pass IFunctionBase here not FunctionCaptureOverloadResolver.
759
    const auto * actions_node = actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), function_capture);
760

761
    if (!result_type->equals(*actions_node->result_type))
762
        throw Exception(ErrorCodes::LOGICAL_ERROR,
763
            "Lambda resolved type {} is not equal to type from actions DAG {}",
764
            result_type, actions_node->result_type);
765

766
    size_t actions_stack_size = actions_stack.size();
767
    for (size_t i = level + 1; i < actions_stack_size; ++i)
768
    {
769
        auto & actions_stack_node = actions_stack[i];
770
        actions_stack_node.addInputColumnIfNecessary(lambda_node_name, result_type);
771
    }
772

773
    return {lambda_node_name, levels};
774
}
775

776
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::makeSetForInFunction(const QueryTreeNodePtr & node)
777
{
778
    const auto & function_node = node->as<FunctionNode &>();
779
    auto in_first_argument = function_node.getArguments().getNodes().at(0);
780
    auto in_second_argument = function_node.getArguments().getNodes().at(1);
781

782
    DataTypes set_element_types;
783

784
    auto in_second_argument_node_type = in_second_argument->getNodeType();
785

786
    bool subquery_or_table =
787
        in_second_argument_node_type == QueryTreeNodeType::QUERY ||
788
        in_second_argument_node_type == QueryTreeNodeType::UNION ||
789
        in_second_argument_node_type == QueryTreeNodeType::TABLE;
790

791
    FutureSetPtr set;
792
    auto set_key = in_second_argument->getTreeHash();
793

794
    if (!subquery_or_table)
795
    {
796
        set_element_types = {in_first_argument->getResultType()};
797
        const auto * left_tuple_type = typeid_cast<const DataTypeTuple *>(set_element_types.front().get());
798
        if (left_tuple_type && left_tuple_type->getElements().size() != 1)
799
            set_element_types = left_tuple_type->getElements();
800

801
        set_element_types = Set::getElementTypes(std::move(set_element_types), planner_context->getQueryContext()->getSettingsRef().transform_null_in);
802
        set = planner_context->getPreparedSets().findTuple(set_key, set_element_types);
803
    }
804
    else
805
    {
806
        set = planner_context->getPreparedSets().findSubquery(set_key);
807
        if (!set)
808
            set = planner_context->getPreparedSets().findStorage(set_key);
809
    }
810

811
    if (!set)
812
        throw Exception(ErrorCodes::LOGICAL_ERROR,
813
            "No set is registered for key {}",
814
            PreparedSets::toString(set_key, set_element_types));
815

816
    ColumnWithTypeAndName column;
817
    column.name = planner_context->createSetKey(in_first_argument->getResultType(), in_second_argument);
818
    column.type = std::make_shared<DataTypeSet>();
819

820
    bool set_is_created = set->get() != nullptr;
821
    auto column_set = ColumnSet::create(1, std::move(set));
822

823
    if (set_is_created)
824
        column.column = ColumnConst::create(std::move(column_set), 1);
825
    else
826
        column.column = std::move(column_set);
827

828
    actions_stack[0].addConstantIfNecessary(column.name, column);
829

830
    size_t actions_stack_size = actions_stack.size();
831
    for (size_t i = 1; i < actions_stack_size; ++i)
832
    {
833
        auto & actions_stack_node = actions_stack[i];
834
        actions_stack_node.addInputConstantColumnIfNecessary(column.name, column);
835
    }
836

837
    return {column.name, Levels(0)};
838
}
839

840
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitIndexHintFunction(const QueryTreeNodePtr & node)
841
{
842
    const auto & function_node = node->as<FunctionNode &>();
843
    auto function_node_name = action_node_name_helper.calculateActionNodeName(node);
844

845
    auto index_hint_actions_dag = std::make_shared<ActionsDAG>();
846
    auto & index_hint_actions_dag_outputs = index_hint_actions_dag->getOutputs();
847
    std::unordered_set<std::string_view> index_hint_actions_dag_output_node_names;
848
    PlannerActionsVisitor actions_visitor(planner_context);
849

850
    for (const auto & argument : function_node.getArguments())
851
    {
852
        auto index_hint_argument_expression_dag_nodes = actions_visitor.visit(index_hint_actions_dag, argument);
853

854
        for (auto & expression_dag_node : index_hint_argument_expression_dag_nodes)
855
        {
856
            if (index_hint_actions_dag_output_node_names.contains(expression_dag_node->result_name))
857
                continue;
858

859
            index_hint_actions_dag_output_node_names.insert(expression_dag_node->result_name);
860
            index_hint_actions_dag_outputs.push_back(expression_dag_node);
861
        }
862
    }
863

864
    auto index_hint_function = std::make_shared<FunctionIndexHint>();
865
    index_hint_function->setActions(std::move(index_hint_actions_dag));
866
    auto index_hint_function_overload_resolver = std::make_shared<FunctionToOverloadResolverAdaptor>(std::move(index_hint_function));
867

868
    size_t index_hint_function_level = actions_stack.size() - 1;
869
    actions_stack[index_hint_function_level].addFunctionIfNecessary(function_node_name, {}, index_hint_function_overload_resolver);
870

871
    return {function_node_name, Levels(index_hint_function_level)};
872
}
873

874
PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node)
875
{
876
    const auto & function_node = node->as<FunctionNode &>();
877
    if (function_node.getFunctionName() == "indexHint")
878
        return visitIndexHintFunction(node);
879

880
    std::optional<NodeNameAndNodeMinLevel> in_function_second_argument_node_name_with_level;
881

882
    if (isNameOfInFunction(function_node.getFunctionName()))
883
        in_function_second_argument_node_name_with_level = makeSetForInFunction(node);
884

885
    auto function_node_name = action_node_name_helper.calculateActionNodeName(node);
886

887
    /* Aggregate functions, window functions, and GROUP BY expressions were already analyzed in the previous steps.
888
     * If we have already visited some expression, we don't need to revisit it or its arguments again.
889
     * For example, the expression from the aggregation step is also present in the projection:
890
     *    SELECT foo(a, b, c) as x FROM table GROUP BY foo(a, b, c)
891
     * In this case we should not analyze `a`, `b`, `c` again.
892
     * Moreover, it can lead to an error if we have arrayJoin in the arguments because it will be calculated twice.
893
     */
894
    bool is_input_node = function_node.isAggregateFunction() || function_node.isWindowFunction()
895
        || actions_stack.front().containsInputNode(function_node_name);
896
    if (is_input_node)
897
    {
898
        size_t actions_stack_size = actions_stack.size();
899

900
        for (size_t i = 0; i < actions_stack_size; ++i)
901
        {
902
            auto & actions_stack_node = actions_stack[i];
903
            actions_stack_node.addInputColumnIfNecessary(function_node_name, function_node.getResultType());
904
        }
905

906
        return {function_node_name, Levels(0)};
907
    }
908

909
    const auto & function_arguments = function_node.getArguments().getNodes();
910
    size_t function_arguments_size = function_arguments.size();
911

912
    Names function_arguments_node_names;
913
    function_arguments_node_names.reserve(function_arguments_size);
914

915
    Levels levels(0);
916
    for (size_t function_argument_index = 0; function_argument_index < function_arguments_size; ++function_argument_index)
917
    {
918
        if (in_function_second_argument_node_name_with_level && function_argument_index == 1)
919
        {
920
            auto & [node_name, node_levels] = *in_function_second_argument_node_name_with_level;
921
            function_arguments_node_names.push_back(std::move(node_name));
922
            levels.add(node_levels);
923
            continue;
924
        }
925

926
        const auto & argument = function_arguments[function_argument_index];
927

928
        if (argument->getNodeType() == QueryTreeNodeType::LAMBDA)
929
        {
930
            auto [node_name, node_levels] = visitLambda(argument);
931
            function_arguments_node_names.push_back(std::move(node_name));
932
            levels.add(node_levels);
933
            continue;
934
        }
935

936
        auto [node_name, node_levels] = visitImpl(argument);
937
        function_arguments_node_names.push_back(std::move(node_name));
938
        levels.add(node_levels);
939
    }
940

941
    ActionsDAG::NodeRawConstPtrs children;
942
    children.reserve(function_arguments_size);
943

944
    size_t level = levels.max();
945
    for (auto & function_argument_node_name : function_arguments_node_names)
946
        children.push_back(actions_stack[level].getNodeOrThrow(function_argument_node_name));
947

948
    if (function_node.getFunctionName() == "arrayJoin")
949
    {
950
        if (level != 0)
951
            throw Exception(ErrorCodes::BAD_ARGUMENTS,
952
                "Expression in arrayJoin cannot depend on lambda argument: {} ",
953
                function_arguments_node_names.at(0));
954

955
        actions_stack[level].addArrayJoinIfNecessary(function_node_name, children.at(0));
956
    }
957
    else
958
    {
959
        actions_stack[level].addFunctionIfNecessary(function_node_name, children, function_node);
960
    }
961

962
    size_t actions_stack_size = actions_stack.size();
963
    for (size_t i = level + 1; i < actions_stack_size; ++i)
964
    {
965
        auto & actions_stack_node = actions_stack[i];
966
        actions_stack_node.addInputColumnIfNecessary(function_node_name, function_node.getResultType());
967
    }
968

969
    return {function_node_name, levels};
970
}
971

972
}
973

974
PlannerActionsVisitor::PlannerActionsVisitor(const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_)
975
    : planner_context(planner_context_)
976
    , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
977
{}
978

979
ActionsDAG::NodeRawConstPtrs PlannerActionsVisitor::visit(ActionsDAGPtr actions_dag, QueryTreeNodePtr expression_node)
980
{
981
    PlannerActionsVisitorImpl actions_visitor_impl(actions_dag, planner_context, use_column_identifier_as_action_node_name);
982
    return actions_visitor_impl.visit(expression_node);
983
}
984

985
String calculateActionNodeName(const QueryTreeNodePtr & node,
986
    const PlannerContext & planner_context,
987
    QueryTreeNodeToName & node_to_name,
988
    bool use_column_identifier_as_action_node_name)
989
{
990
    ActionNodeNameHelper helper(node_to_name, planner_context, use_column_identifier_as_action_node_name);
991
    return helper.calculateActionNodeName(node);
992
}
993

994
String calculateActionNodeName(const QueryTreeNodePtr & node, const PlannerContext & planner_context, bool use_column_identifier_as_action_node_name)
995
{
996
    QueryTreeNodeToName empty_map;
997
    ActionNodeNameHelper helper(empty_map, planner_context, use_column_identifier_as_action_node_name);
998
    return helper.calculateActionNodeName(node);
999
}
1000

1001
String calculateConstantActionNodeName(const Field & constant_literal, const DataTypePtr & constant_type)
1002
{
1003
    return ActionNodeNameHelper::calculateConstantActionNodeName(constant_literal, constant_type);
1004
}
1005

1006
String calculateConstantActionNodeName(const Field & constant_literal)
1007
{
1008
    return ActionNodeNameHelper::calculateConstantActionNodeName(constant_literal);
1009
}
1010

1011
String calculateWindowNodeActionName(const QueryTreeNodePtr & node,
1012
    const PlannerContext & planner_context,
1013
    QueryTreeNodeToName & node_to_name,
1014
    bool use_column_identifier_as_action_node_name)
1015
{
1016
    ActionNodeNameHelper helper(node_to_name, planner_context, use_column_identifier_as_action_node_name);
1017
    return helper.calculateWindowNodeActionName(node);
1018
}
1019

1020
String calculateWindowNodeActionName(const QueryTreeNodePtr & node, const PlannerContext & planner_context, bool use_column_identifier_as_action_node_name)
1021
{
1022
    QueryTreeNodeToName empty_map;
1023
    ActionNodeNameHelper helper(empty_map, planner_context, use_column_identifier_as_action_node_name);
1024
    return helper.calculateWindowNodeActionName(node);
1025
}
1026

1027
}
1028