ClickHouse

1
#include "Loggers.h"
2

3
#include "OwnFormattingChannel.h"
4
#include "OwnPatternFormatter.h"
5
#include "OwnSplitChannel.h"
6

7
#include <iostream>
8
#include <sstream>
9

10
#include <Poco/ConsoleChannel.h>
11
#include <Poco/Logger.h>
12
#include <Poco/Net/RemoteSyslogChannel.h>
13
#include <Poco/SyslogChannel.h>
14
#include <Poco/Util/AbstractConfiguration.h>
15

16
#ifndef WITHOUT_TEXT_LOG
17
    #include <Interpreters/TextLog.h>
18
#endif
19

20
#include <filesystem>
21

22
namespace fs = std::filesystem;
23

24
namespace DB
25
{
26
    class SensitiveDataMasker;
27

28
namespace ErrorCodes
29
{
30
    extern const int BAD_ARGUMENTS;
31
}
32

33
}
34

35

36
// TODO: move to libcommon
37
static std::string createDirectory(const std::string & file)
38
{
39
    auto path = fs::path(file).parent_path();
40
    if (path.empty())
41
        return "";
42
    fs::create_directories(path);
43
    return path;
44
}
45

46
static std::string renderFileNameTemplate(time_t now, const std::string & file_path)
47
{
48
    fs::path path{file_path};
49
    std::tm buf;
50
    localtime_r(&now, &buf);
51
    std::ostringstream ss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
52
    ss << std::put_time(&buf, path.filename().c_str());
53
    return path.replace_filename(ss.str());
54
}
55

56
/// NOLINTBEGIN(readability-static-accessed-through-instance)
57

58
void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger /*_root*/, const std::string & cmd_name)
59
{
60
    auto current_logger = config.getString("logger", "");
61
    if (config_logger.has_value() && *config_logger == current_logger)
62
        return;
63

64
    config_logger = current_logger;
65

66
    bool is_daemon = config.getBool("application.runAsDaemon", false);
67

68
    /// Split logs to ordinary log, error log, syslog and console.
69
    /// Use extended interface of Channel for more comprehensive logging.
70
    split = new DB::OwnSplitChannel();
71

72
    auto log_level_string = config.getString("logger.level", "trace");
73

74
    /// different channels (log, console, syslog) may have different loglevels configured
75
    /// The maximum (the most verbose) of those will be used as default for Poco loggers
76
    int max_log_level = 0;
77

78
    time_t now = std::time({});
79

80
    const auto log_path_prop = config.getString("logger.log", "");
81
    if (!log_path_prop.empty())
82
    {
83
        const auto log_path = renderFileNameTemplate(now, log_path_prop);
84
        createDirectory(log_path);
85

86
        std::string ext;
87
        if (config.getRawString("logger.stream_compress", "false") == "true")
88
            ext = ".lz4";
89

90
        std::cerr << "Logging " << log_level_string << " to " << log_path << ext << std::endl;
91

92
        auto log_level = Poco::Logger::parseLevel(log_level_string);
93
        if (log_level > max_log_level)
94
        {
95
            max_log_level = log_level;
96
        }
97

98
        // Set up two channel chains.
99
        log_file = new Poco::FileChannel;
100
        log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(log_path));
101
        log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
102
        log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
103
        log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
104
        log_file->setProperty(Poco::FileChannel::PROP_STREAMCOMPRESS, config.getRawString("logger.stream_compress", "false"));
105
        log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
106
        log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
107
        log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false"));
108
        log_file->open();
109

110
        Poco::AutoPtr<OwnPatternFormatter> pf;
111

112
        if (config.getString("logger.formatting.type", "") == "json")
113
            pf = new OwnJSONPatternFormatter(config);
114
        else
115
            pf = new OwnPatternFormatter;
116

117
        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, log_file);
118
        log->setLevel(log_level);
119
        split->addChannel(log, "log");
120
    }
121

122
    const auto errorlog_path_prop = config.getString("logger.errorlog", "");
123
    if (!errorlog_path_prop.empty())
124
    {
125
        const auto errorlog_path = renderFileNameTemplate(now, errorlog_path_prop);
126
        createDirectory(errorlog_path);
127

128
        // NOTE: we don't use notice & critical in the code, so in practice error log collects fatal & error & warning.
129
        // (!) Warnings are important, they require attention and should never be silenced / ignored.
130
        auto errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice"));
131
        if (errorlog_level > max_log_level)
132
        {
133
            max_log_level = errorlog_level;
134
        }
135

136
        std::string ext;
137
        if (config.getRawString("logger.stream_compress", "false") == "true")
138
            ext = ".lz4";
139

140
        std::cerr << "Logging errors to " << errorlog_path << ext << std::endl;
141

142
        error_log_file = new Poco::FileChannel;
143
        error_log_file->setProperty(Poco::FileChannel::PROP_PATH, fs::weakly_canonical(errorlog_path));
144
        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATION, config.getRawString("logger.size", "100M"));
145
        error_log_file->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number");
146
        error_log_file->setProperty(Poco::FileChannel::PROP_COMPRESS, config.getRawString("logger.compress", "true"));
147
        error_log_file->setProperty(Poco::FileChannel::PROP_STREAMCOMPRESS, config.getRawString("logger.stream_compress", "false"));
148
        error_log_file->setProperty(Poco::FileChannel::PROP_PURGECOUNT, config.getRawString("logger.count", "1"));
149
        error_log_file->setProperty(Poco::FileChannel::PROP_FLUSH, config.getRawString("logger.flush", "true"));
150
        error_log_file->setProperty(Poco::FileChannel::PROP_ROTATEONOPEN, config.getRawString("logger.rotateOnOpen", "false"));
151

152
        Poco::AutoPtr<OwnPatternFormatter> pf;
153

154
        if (config.getString("logger.formatting.type", "") == "json")
155
            pf = new OwnJSONPatternFormatter(config);
156
        else
157
            pf = new OwnPatternFormatter;
158

159
        Poco::AutoPtr<DB::OwnFormattingChannel> errorlog = new DB::OwnFormattingChannel(pf, error_log_file);
160
        errorlog->setLevel(errorlog_level);
161
        errorlog->open();
162
        split->addChannel(errorlog, "errorlog");
163
    }
164

165
    if (config.getBool("logger.use_syslog", false))
166
    {
167
        auto syslog_level = Poco::Logger::parseLevel(config.getString("logger.syslog_level", log_level_string));
168
        if (syslog_level > max_log_level)
169
        {
170
            max_log_level = syslog_level;
171
        }
172

173
        if (config.has("logger.syslog.address"))
174
        {
175
            syslog_channel = new Poco::Net::RemoteSyslogChannel();
176
            // syslog address
177
            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_LOGHOST, config.getString("logger.syslog.address"));
178
            if (config.has("logger.syslog.hostname"))
179
            {
180
                syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_HOST, config.getString("logger.syslog.hostname"));
181
            }
182
            syslog_channel->setProperty(Poco::Net::RemoteSyslogChannel::PROP_FORMAT, config.getString("logger.syslog.format", "syslog"));
183
            syslog_channel->setProperty(
184
                Poco::Net::RemoteSyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_USER"));
185
        }
186
        else
187
        {
188
            syslog_channel = new Poco::SyslogChannel();
189
            syslog_channel->setProperty(Poco::SyslogChannel::PROP_NAME, cmd_name);
190
            syslog_channel->setProperty(Poco::SyslogChannel::PROP_OPTIONS, config.getString("logger.syslog.options", "LOG_CONS|LOG_PID"));
191
            syslog_channel->setProperty(Poco::SyslogChannel::PROP_FACILITY, config.getString("logger.syslog.facility", "LOG_DAEMON"));
192
        }
193
        syslog_channel->open();
194

195
        Poco::AutoPtr<OwnPatternFormatter> pf;
196

197
        if (config.getString("logger.formatting.type", "") == "json")
198
            pf = new OwnJSONPatternFormatter(config);
199
        else
200
            pf = new OwnPatternFormatter;
201

202
        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, syslog_channel);
203
        log->setLevel(syslog_level);
204

205
        split->addChannel(log, "syslog");
206
    }
207

208
    bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
209
    bool color_logs_by_default = isatty(STDERR_FILENO);
210

211
    if (config.getBool("logger.console", false)
212
        || (!config.hasProperty("logger.console") && !is_daemon && should_log_to_console))
213
    {
214
        bool color_enabled = config.getBool("logger.color_terminal", color_logs_by_default);
215

216
        auto console_log_level_string = config.getString("logger.console_log_level", log_level_string);
217
        auto console_log_level = Poco::Logger::parseLevel(console_log_level_string);
218
        if (console_log_level > max_log_level)
219
        {
220
            max_log_level = console_log_level;
221
        }
222

223
        Poco::AutoPtr<OwnPatternFormatter> pf;
224
        if (config.getString("logger.formatting.type", "") == "json")
225
            pf = new OwnJSONPatternFormatter(config);
226
        else
227
            pf = new OwnPatternFormatter(color_enabled);
228
        Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);
229
        log->setLevel(console_log_level);
230
        split->addChannel(log, "console");
231
    }
232

233
    split->open();
234
    logger.close();
235

236
    logger.setChannel(split);
237
    logger.setLevel(max_log_level);
238

239
    // Global logging level and channel (it can be overridden for specific loggers).
240
    logger.root().setLevel(max_log_level);
241
    logger.root().setChannel(logger.getChannel());
242

243
    // Set level and channel to all already created loggers
244
    std::vector<std::string> names;
245
    logger.names(names);
246

247
    for (const auto & name : names)
248
    {
249
        logger.get(name).setLevel(max_log_level);
250
        logger.get(name).setChannel(split);
251
    }
252

253
    // Explicitly specified log levels for specific loggers.
254
    {
255
        Poco::Util::AbstractConfiguration::Keys loggers_level;
256
        config.keys("logger.levels", loggers_level);
257

258
        if (!loggers_level.empty())
259
        {
260
            for (const auto & key : loggers_level)
261
            {
262
                if (key == "logger" || key.starts_with("logger["))
263
                {
264
                    const std::string name(config.getString("logger.levels." + key + ".name"));
265
                    const std::string level(config.getString("logger.levels." + key + ".level"));
266
                    logger.root().get(name).setLevel(level);
267
                }
268
                else
269
                {
270
                    // Legacy syntax
271
                    const std::string level(config.getString("logger.levels." + key, "trace"));
272
                    logger.root().get(key).setLevel(level);
273
                }
274
            }
275
        }
276
    }
277
#ifndef WITHOUT_TEXT_LOG
278
    if (config.has("text_log"))
279
    {
280
        String text_log_level_str = config.getString("text_log.level", "trace");
281
        int text_log_level = Poco::Logger::parseLevel(text_log_level_str);
282

283
        DB::SystemLogQueueSettings log_settings;
284
        log_settings.flush_interval_milliseconds = config.getUInt64("text_log.flush_interval_milliseconds",
285
                                                                    DB::TextLog::getDefaultFlushIntervalMilliseconds());
286

287
        log_settings.max_size_rows = config.getUInt64("text_log.max_size_rows",
288
                                                      DB::TextLog::getDefaultMaxSize());
289

290
        if (log_settings.max_size_rows< 1)
291
            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "text_log.max_size_rows {} should be 1 at least",
292
                                log_settings.max_size_rows);
293

294
        log_settings.reserved_size_rows = config.getUInt64("text_log.reserved_size_rows", DB::TextLog::getDefaultReservedSize());
295

296
        if (log_settings.max_size_rows < log_settings.reserved_size_rows)
297
        {
298
            throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS,
299
                                "text_log.max_size {0} should be greater or equal to text_log.reserved_size_rows {1}",
300
                                log_settings.max_size_rows,
301
                                log_settings.reserved_size_rows);
302
        }
303

304
        log_settings.buffer_size_rows_flush_threshold = config.getUInt64("text_log.buffer_size_rows_flush_threshold",
305
                                                                         log_settings.max_size_rows / 2);
306

307
        log_settings.notify_flush_on_crash = config.getBool("text_log.flush_on_crash",
308
                                                            DB::TextLog::shouldNotifyFlushOnCrash());
309

310
        log_settings.turn_off_logger = DB::TextLog::shouldTurnOffLogger();
311

312
        log_settings.database = config.getString("text_log.database", "system");
313
        log_settings.table = config.getString("text_log.table", "text_log");
314

315
        split->addTextLog(DB::TextLog::getLogQueue(log_settings), text_log_level);
316
    }
317
#endif
318
}
319

320
void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Logger & logger)
321
{
322
    int max_log_level = 0;
323

324
    const auto log_level_string = config.getString("logger.level", "trace");
325
    int log_level = Poco::Logger::parseLevel(log_level_string);
326
    if (log_level > max_log_level)
327
        max_log_level = log_level;
328

329
    if (log_file)
330
        split->setLevel("log", log_level);
331

332
    // Set level to console
333
    bool is_daemon = config.getBool("application.runAsDaemon", false);
334
    bool should_log_to_console = isatty(STDIN_FILENO) || isatty(STDERR_FILENO);
335
    if (config.getBool("logger.console", false)
336
        || (!config.hasProperty("logger.console") && !is_daemon && should_log_to_console))
337
        split->setLevel("console", log_level);
338
    else
339
        split->setLevel("console", 0);
340

341
    // Set level to errorlog
342
    if (error_log_file)
343
    {
344
        int errorlog_level = Poco::Logger::parseLevel(config.getString("logger.errorlog_level", "notice"));
345
        if (errorlog_level > max_log_level)
346
            max_log_level = errorlog_level;
347
        split->setLevel("errorlog", errorlog_level);
348
    }
349

350
    // Set level to syslog
351
    int syslog_level = 0;
352
    if (config.getBool("logger.use_syslog", false))
353
    {
354
        syslog_level = Poco::Logger::parseLevel(config.getString("logger.syslog_level", log_level_string));
355
        if (syslog_level > max_log_level)
356
            max_log_level = syslog_level;
357
    }
358
    split->setLevel("syslog", syslog_level);
359

360
    // Global logging level (it can be overridden for specific loggers).
361
    logger.setLevel(max_log_level);
362

363
    // Set level to all already created loggers
364
    std::vector<std::string> names;
365

366
    logger.root().names(names);
367
    for (const auto & name : names)
368
        logger.root().get(name).setLevel(max_log_level);
369

370
    logger.root().setLevel(max_log_level);
371

372
    // Explicitly specified log levels for specific loggers.
373
    {
374
        Poco::Util::AbstractConfiguration::Keys loggers_level;
375
        config.keys("logger.levels", loggers_level);
376

377
        if (!loggers_level.empty())
378
        {
379
            for (const auto & key : loggers_level)
380
            {
381
                if (key == "logger" || key.starts_with("logger["))
382
                {
383
                    const std::string name(config.getString("logger.levels." + key + ".name"));
384
                    const std::string level(config.getString("logger.levels." + key + ".level"));
385
                    logger.root().get(name).setLevel(level);
386
                }
387
                else
388
                {
389
                    // Legacy syntax
390
                    const std::string level(config.getString("logger.levels." + key, "trace"));
391
                    logger.root().get(key).setLevel(level);
392
                }
393
            }
394
        }
395
    }
396
}
397

398
/// NOLINTEND(readability-static-accessed-through-instance)
399

400
void Loggers::closeLogs(Poco::Logger & logger)
401
{
402
    if (log_file)
403
        log_file->close();
404
    if (error_log_file)
405
        error_log_file->close();
406
    // Shouldn't syslog_channel be closed here too?
407

408
    if (!log_file)
409
        logger.warning("Logging to console but received signal to close log file (ignoring).");
410
}
411