pg_probackup

1
/*-------------------------------------------------------------------------
2
 *
3
 * util.c: log messages to log file or stderr, and misc code.
4
 *
5
 * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
6
 * Portions Copyright (c) 2015-2021, Postgres Professional
7
 *
8
 *-------------------------------------------------------------------------
9
 */
10

11
#include "pg_probackup.h"
12

13
#include <time.h>
14

15
#include <unistd.h>
16

17
#include <sys/stat.h>
18

19
static const char *statusName[] =
20
{
21
	"UNKNOWN",
22
	"OK",
23
	"ERROR",
24
	"RUNNING",
25
	"MERGING",
26
	"MERGED",
27
	"DELETING",
28
	"DELETED",
29
	"DONE",
30
	"ORPHAN",
31
	"CORRUPT"
32
};
33

34
const char *
35
base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize])
36
{
37
	const char	base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
38
	char	buffer[base36bufsize];
39
	char   *p;
40

41
	p = &buffer[sizeof(buffer)-1];
42
	*p = '\0';
43
	do {
44
		*(--p) = base36[value % 36];
45
	} while (value /= 36);
46

47
	/* I know, it doesn't look safe */
48
	strncpy(buf, p, base36bufsize);
49

50
	return buf;
51
}
52

53
long unsigned int
54
base36dec(const char *text)
55
{
56
	return strtoul(text, NULL, 36);
57
}
58

59
static void
60
checkControlFile(ControlFileData *ControlFile)
61
{
62
	pg_crc32c   crc;
63

64
	/* Calculate CRC */
65
	INIT_CRC32C(crc);
66
	COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc));
67
	FIN_CRC32C(crc);
68

69
	/* Then compare it */
70
	if (!EQ_CRC32C(crc, ControlFile->crc))
71
		elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n"
72
			 "Either the file is corrupt, or it has a different layout than this program\n"
73
			 "is expecting. The results below are untrustworthy.");
74

75
	if ((ControlFile->pg_control_version % 65536 == 0 || ControlFile->pg_control_version % 65536 > 10000) &&
76
			ControlFile->pg_control_version / 65536 != 0)
77
		elog(ERROR, "Possible byte ordering mismatch\n"
78
			 "The byte ordering used to store the pg_control file might not match the one\n"
79
			 "used by this program. In that case the results below would be incorrect, and\n"
80
			 "the PostgreSQL installation would be incompatible with this data directory.");
81
}
82

83
/*
84
 * Verify control file contents in the buffer src, and copy it to *ControlFile.
85
 */
86
static void
87
digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
88
{
89
#if PG_VERSION_NUM >= 100000
90
	int			ControlFileSize = PG_CONTROL_FILE_SIZE;
91
#else
92
	int			ControlFileSize = PG_CONTROL_SIZE;
93
#endif
94

95
	if (size != ControlFileSize)
96
		elog(ERROR, "Unexpected control file size %d, expected %d",
97
			 (int) size, ControlFileSize);
98

99
	memcpy(ControlFile, src, sizeof(ControlFileData));
100

101
	/* Additional checks on control file */
102
	checkControlFile(ControlFile);
103
}
104

105
/*
106
 * Write ControlFile to pg_control
107
 */
108
static void
109
writeControlFile(ControlFileData *ControlFile, const char *path, fio_location location)
110
{
111
	int			fd;
112
	char       *buffer = NULL;
113

114
#if PG_VERSION_NUM >= 100000
115
	int			ControlFileSize = PG_CONTROL_FILE_SIZE;
116
#else
117
	int			ControlFileSize = PG_CONTROL_SIZE;
118
#endif
119

120
	/* copy controlFileSize */
121
	buffer = pg_malloc0(ControlFileSize);
122
	memcpy(buffer, ControlFile, sizeof(ControlFileData));
123

124
	/* Write pg_control */
125
	fd = fio_open(path,
126
				  O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, location);
127

128
	if (fd < 0)
129
		elog(ERROR, "Failed to open file: %s", path);
130

131
	if (fio_write(fd, buffer, ControlFileSize) != ControlFileSize)
132
		elog(ERROR, "Failed to overwrite file: %s", path);
133

134
	if (fio_flush(fd) != 0)
135
		elog(ERROR, "Failed to sync file: %s", path);
136

137
	fio_close(fd);
138
	pg_free(buffer);
139
}
140

141
/*
142
 * Utility shared by backup and restore to fetch the current timeline
143
 * used by a node.
144
 */
145
TimeLineID
146
get_current_timeline(PGconn *conn)
147
{
148

149
	PGresult   *res;
150
	TimeLineID tli = 0;
151
	char	   *val;
152

153
	res = pgut_execute_extended(conn,
154
				   "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()", 0, NULL, true, true);
155

156
	if (PQresultStatus(res) == PGRES_TUPLES_OK)
157
		val = PQgetvalue(res, 0, 0);
158
	else
159
		return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false);
160

161
	if (!parse_uint32(val, &tli, 0))
162
	{
163
		PQclear(res);
164
		elog(WARNING, "Invalid value of timeline_id %s", val);
165

166
		/* TODO 3.0 remove it and just error out */
167
		return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false);
168
	}
169

170
	return tli;
171
}
172

173
/* Get timeline from pg_control file */
174
TimeLineID
175
get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe)
176
{
177
	ControlFileData ControlFile;
178
	char       *buffer;
179
	size_t      size;
180

181
	/* First fetch file... */
182
	buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size,
183
					   safe, location);
184
	if (safe && buffer == NULL)
185
		return 0;
186

187
	digestControlFile(&ControlFile, buffer, size);
188
	pg_free(buffer);
189

190
	return ControlFile.checkPointCopy.ThisTimeLineID;
191
}
192

193
/*
194
 * Get last check point record ptr from pg_tonrol.
195
 */
196
XLogRecPtr
197
get_checkpoint_location(PGconn *conn)
198
{
199
#if PG_VERSION_NUM >= 90600
200
	PGresult   *res;
201
	uint32		lsn_hi;
202
	uint32		lsn_lo;
203
	XLogRecPtr	lsn;
204

205
#if PG_VERSION_NUM >= 100000
206
	res = pgut_execute(conn,
207
					   "SELECT checkpoint_lsn FROM pg_catalog.pg_control_checkpoint()",
208
					   0, NULL);
209
#else
210
	res = pgut_execute(conn,
211
					   "SELECT checkpoint_location FROM pg_catalog.pg_control_checkpoint()",
212
					   0, NULL);
213
#endif
214
	XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);
215
	PQclear(res);
216
	/* Calculate LSN */
217
	lsn = ((uint64) lsn_hi) << 32 | lsn_lo;
218

219
	return lsn;
220
#else
221
	char	   *buffer;
222
	size_t		size;
223
	ControlFileData ControlFile;
224

225
	buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
226
	digestControlFile(&ControlFile, buffer, size);
227
	pg_free(buffer);
228

229
	return ControlFile.checkPoint;
230
#endif
231
}
232

233
uint64
234
get_system_identifier(const char *pgdata_path, fio_location location, bool safe)
235
{
236
	ControlFileData ControlFile;
237
	char	   *buffer;
238
	size_t		size;
239

240
	/* First fetch file... */
241
	buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location);
242
	if (safe && buffer == NULL)
243
		return 0;
244
	digestControlFile(&ControlFile, buffer, size);
245
	pg_free(buffer);
246

247
	return ControlFile.system_identifier;
248
}
249

250
uint64
251
get_remote_system_identifier(PGconn *conn)
252
{
253
#if PG_VERSION_NUM >= 90600
254
	PGresult   *res;
255
	uint64		system_id_conn;
256
	char	   *val;
257

258
	res = pgut_execute(conn,
259
					   "SELECT system_identifier FROM pg_catalog.pg_control_system()",
260
					   0, NULL);
261
	val = PQgetvalue(res, 0, 0);
262
	if (!parse_uint64(val, &system_id_conn, 0))
263
	{
264
		PQclear(res);
265
		elog(ERROR, "%s is not system_identifier", val);
266
	}
267
	PQclear(res);
268

269
	return system_id_conn;
270
#else
271
	char	   *buffer;
272
	size_t		size;
273
	ControlFileData ControlFile;
274

275
	buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
276
	digestControlFile(&ControlFile, buffer, size);
277
	pg_free(buffer);
278

279
	return ControlFile.system_identifier;
280
#endif
281
}
282

283
uint32
284
get_xlog_seg_size(const char *pgdata_path)
285
{
286
#if PG_VERSION_NUM >= 110000
287
	ControlFileData ControlFile;
288
	char	   *buffer;
289
	size_t		size;
290

291
	/* First fetch file... */
292
	buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
293
	digestControlFile(&ControlFile, buffer, size);
294
	pg_free(buffer);
295

296
	return ControlFile.xlog_seg_size;
297
#else
298
	return (uint32) XLOG_SEG_SIZE;
299
#endif
300
}
301

302
uint32
303
get_data_checksum_version(bool safe)
304
{
305
	ControlFileData ControlFile;
306
	char	   *buffer;
307
	size_t		size;
308

309
	/* First fetch file... */
310
	buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size,
311
					   safe, FIO_DB_HOST);
312
	if (buffer == NULL)
313
		return 0;
314
	digestControlFile(&ControlFile, buffer, size);
315
	pg_free(buffer);
316

317
	return ControlFile.data_checksum_version;
318
}
319

320
pg_crc32c
321
get_pgcontrol_checksum(const char *pgdata_path)
322
{
323
	ControlFileData ControlFile;
324
	char	   *buffer;
325
	size_t		size;
326

327
	/* First fetch file... */
328
	buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST);
329

330
	digestControlFile(&ControlFile, buffer, size);
331
	pg_free(buffer);
332

333
	return ControlFile.crc;
334
}
335

336
void
337
get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo)
338
{
339
	ControlFileData ControlFile;
340
	char	   *buffer;
341
	size_t		size;
342

343
	/* First fetch file... */
344
	buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, pgdata_location);
345

346
	digestControlFile(&ControlFile, buffer, size);
347
	pg_free(buffer);
348

349
	redo->lsn = ControlFile.checkPointCopy.redo;
350
	redo->tli = ControlFile.checkPointCopy.ThisTimeLineID;
351

352
	if (ControlFile.minRecoveryPoint > 0 &&
353
		ControlFile.minRecoveryPoint < redo->lsn)
354
	{
355
		redo->lsn = ControlFile.minRecoveryPoint;
356
		redo->tli = ControlFile.minRecoveryPointTLI;
357
	}
358

359
	if (ControlFile.backupStartPoint > 0 &&
360
		ControlFile.backupStartPoint < redo->lsn)
361
	{
362
		redo->lsn = ControlFile.backupStartPoint;
363
		redo->tli = ControlFile.checkPointCopy.ThisTimeLineID;
364
	}
365

366
	redo->checksum_version = ControlFile.data_checksum_version;
367
}
368

369
/*
370
 * Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint
371
 * 'as-is' is not to be trusted.
372
 */
373
void
374
set_min_recovery_point(pgFile *file, const char *backup_path,
375
					   XLogRecPtr stop_backup_lsn)
376
{
377
	ControlFileData ControlFile;
378
	char       *buffer;
379
	size_t      size;
380
	char		fullpath[MAXPGPATH];
381

382
	/* First fetch file content */
383
	buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
384
	digestControlFile(&ControlFile, buffer, size);
385

386
	elog(LOG, "Current minRecPoint %X/%X",
387
		(uint32) (ControlFile.minRecoveryPoint  >> 32),
388
		(uint32) ControlFile.minRecoveryPoint);
389

390
	elog(LOG, "Setting minRecPoint to %X/%X",
391
		(uint32) (stop_backup_lsn  >> 32),
392
		(uint32) stop_backup_lsn);
393

394
	ControlFile.minRecoveryPoint = stop_backup_lsn;
395

396
	/* Update checksum in pg_control header */
397
	INIT_CRC32C(ControlFile.crc);
398
	COMP_CRC32C(ControlFile.crc, (char *) &ControlFile,
399
				offsetof(ControlFileData, crc));
400
	FIN_CRC32C(ControlFile.crc);
401

402
	/* overwrite pg_control */
403
	join_path_components(fullpath, backup_path, XLOG_CONTROL_FILE);
404
	writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST);
405

406
	/* Update pg_control checksum in backup_list */
407
	file->crc = ControlFile.crc;
408

409
	pg_free(buffer);
410
}
411

412
/*
413
 * Copy pg_control file to backup. We do not apply compression to this file.
414
 */
415
void
416
copy_pgcontrol_file(const char *from_fullpath, fio_location from_location,
417
					const char *to_fullpath, fio_location to_location, pgFile *file)
418
{
419
	ControlFileData ControlFile;
420
	char	   *buffer;
421
	size_t		size;
422

423
	buffer = slurpFile(from_fullpath, "", &size, false, from_location);
424

425
	digestControlFile(&ControlFile, buffer, size);
426

427
	file->crc = ControlFile.crc;
428
	file->read_size = size;
429
	file->write_size = size;
430
	file->uncompressed_size = size;
431

432
	writeControlFile(&ControlFile, to_fullpath, to_location);
433

434
	pg_free(buffer);
435
}
436

437
/*
438
 * Parse string representation of the server version.
439
 */
440
uint32
441
parse_server_version(const char *server_version_str)
442
{
443
	int			nfields;
444
	uint32		result = 0;
445
	int			major_version = 0;
446
	int			minor_version = 0;
447

448
	nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version);
449
	if (nfields == 2)
450
	{
451
		/* Server version lower than 10 */
452
		if (major_version > 10)
453
			elog(ERROR, "Server version format doesn't match major version %d", major_version);
454
		result = major_version * 10000 + minor_version * 100;
455
	}
456
	else if (nfields == 1)
457
	{
458
		if (major_version < 10)
459
			elog(ERROR, "Server version format doesn't match major version %d", major_version);
460
		result = major_version * 10000;
461
	}
462
	else
463
		elog(ERROR, "Unknown server version format %s", server_version_str);
464

465
	return result;
466
}
467

468
/*
469
 * Parse string representation of the program version.
470
 */
471
uint32
472
parse_program_version(const char *program_version)
473
{
474
	int			nfields;
475
	int			major = 0,
476
				minor = 0,
477
				micro = 0;
478
	uint32		result = 0;
479

480
	if (program_version == NULL || program_version[0] == '\0')
481
		return 0;
482

483
	nfields = sscanf(program_version, "%d.%d.%d", &major, &minor, &micro);
484
	if (nfields == 3)
485
		result = major * 10000 + minor * 100 + micro;
486
	else
487
		elog(ERROR, "Unknown program version format %s", program_version);
488

489
	return result;
490
}
491

492
const char *
493
status2str(BackupStatus status)
494
{
495
	if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status)
496
		return "UNKNOWN";
497

498
	return statusName[status];
499
}
500

501
const char *
502
status2str_color(BackupStatus status)
503
{
504
	char *status_str = pgut_malloc(20);
505

506
	/* UNKNOWN */
507
	if (status == BACKUP_STATUS_INVALID)
508
		snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, "UNKNOWN", TC_RESET);
509
	/* CORRUPT, ERROR and ORPHAN */
510
	else if (status == BACKUP_STATUS_CORRUPT || status == BACKUP_STATUS_ERROR ||
511
			 status == BACKUP_STATUS_ORPHAN)
512
		snprintf(status_str, 20, "%s%s%s", TC_RED_BOLD, statusName[status], TC_RESET);
513
	/* MERGING, MERGED, DELETING and DELETED */
514
	else if (status == BACKUP_STATUS_MERGING || status == BACKUP_STATUS_MERGED ||
515
			 status == BACKUP_STATUS_DELETING || status == BACKUP_STATUS_DELETED)
516
		snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, statusName[status], TC_RESET);
517
	/* OK and DONE */
518
	else
519
		snprintf(status_str, 20, "%s%s%s", TC_GREEN_BOLD, statusName[status], TC_RESET);
520

521
	return status_str;
522
}
523

524
BackupStatus
525
str2status(const char *status)
526
{
527
	BackupStatus i;
528

529
	for (i = BACKUP_STATUS_INVALID; i <= BACKUP_STATUS_CORRUPT; i++)
530
	{
531
		if (pg_strcasecmp(status, statusName[i]) == 0) return i;
532
	}
533

534
	return BACKUP_STATUS_INVALID;
535
}
536

537
bool
538
datapagemap_is_set(datapagemap_t *map, BlockNumber blkno)
539
{
540
	int			offset;
541
	int			bitno;
542

543
	offset = blkno / 8;
544
	bitno = blkno % 8;
545

546
	return (map->bitmapsize <= offset) ? false : (map->bitmap[offset] & (1 << bitno)) != 0;
547
}
548

549
/*
550
 * A debugging aid. Prints out the contents of the page map.
551
 */
552
void
553
datapagemap_print_debug(datapagemap_t *map)
554
{
555
	datapagemap_iterator_t *iter;
556
	BlockNumber blocknum;
557

558
	iter = datapagemap_iterate(map);
559
	while (datapagemap_next(iter, &blocknum))
560
		elog(VERBOSE, "  block %u", blocknum);
561

562
	pg_free(iter);
563
}
564

565
const char*
566
backup_id_of(pgBackup *backup)
567
{
568
	/* Change this Assert when backup_id will not be bound to start_time */
569
	Assert(backup->backup_id == backup->start_time || backup->start_time == 0);
570

571
	if (backup->backup_id_encoded[0] == '\x00')
572
	{
573
		base36enc_to(backup->backup_id, backup->backup_id_encoded);
574
	}
575
	return backup->backup_id_encoded;
576
}
577

578
void
579
reset_backup_id(pgBackup *backup)
580
{
581
	backup->backup_id = INVALID_BACKUP_ID;
582
	memset(backup->backup_id_encoded, 0, sizeof(backup->backup_id_encoded));
583
}
584