pg_probackup

1
/*-------------------------------------------------------------------------
2
 *
3
 * validate.c: validate backup files.
4
 *
5
 * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
6
 * Portions Copyright (c) 2015-2019, Postgres Professional
7
 *
8
 *-------------------------------------------------------------------------
9
 */
10

11
#include "pg_probackup.h"
12

13
#include <sys/stat.h>
14
#include <dirent.h>
15

16
#include "utils/thread.h"
17

18
static void *pgBackupValidateFiles(void *arg);
19
static void do_validate_instance(InstanceState *instanceState);
20

21
static bool corrupted_backup_found = false;
22
static bool skipped_due_to_lock = false;
23

24
typedef struct
25
{
26
	const char *base_path;
27
	parray		*files;
28
	bool		corrupted;
29
	XLogRecPtr 	stop_lsn;
30
	uint32		checksum_version;
31
	uint32		backup_version;
32
	BackupMode	backup_mode;
33
	parray		*dbOid_exclude_list;
34
	const char	*external_prefix;
35
	HeaderMap   *hdr_map;
36

37
	/*
38
	 * Return value from the thread.
39
	 * 0 means there is no error, 1 - there is an error.
40
	 */
41
	int			ret;
42
} validate_files_arg;
43

44
/*
45
 * Validate backup files.
46
 * TODO: partial validation.
47
 */
48
void
49
pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
50
{
51
	char		external_prefix[MAXPGPATH];
52
	parray	   *files = NULL;
53
	bool		corrupted = false;
54
	bool		validation_isok = true;
55
	/* arrays with meta info for multi threaded validate */
56
	pthread_t  *threads;
57
	validate_files_arg *threads_args;
58
	int			i;
59
//	parray		*dbOid_exclude_list = NULL;
60

61
	/* Check backup program version */
62
	if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION))
63
		elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. "
64
			"pg_probackup do not guarantee to be forward compatible. "
65
			"Please upgrade pg_probackup binary.",
66
				PROGRAM_VERSION, backup_id_of(backup), backup->program_version);
67

68
	/* Check backup server version */
69
	if (strcmp(backup->server_version, PG_MAJORVERSION) != 0)
70
        elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary "
71
					"compiled with server version %s",
72
                backup_id_of(backup), backup->server_version, PG_MAJORVERSION);
73

74
	if (backup->status == BACKUP_STATUS_RUNNING)
75
	{
76
		elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation",
77
			 backup_id_of(backup), status2str(backup->status));
78
		write_backup_status(backup, BACKUP_STATUS_ERROR, true);
79
		corrupted_backup_found = true;
80
		return;
81
	}
82

83
	/* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */
84
	if (backup->status != BACKUP_STATUS_OK &&
85
		backup->status != BACKUP_STATUS_DONE &&
86
		backup->status != BACKUP_STATUS_ORPHAN &&
87
		backup->status != BACKUP_STATUS_MERGING &&
88
		backup->status != BACKUP_STATUS_CORRUPT)
89
	{
90
		elog(WARNING, "Backup %s has status %s. Skip validation.",
91
					backup_id_of(backup), status2str(backup->status));
92
		corrupted_backup_found = true;
93
		return;
94
	}
95

96
	/* additional sanity */
97
	if (backup->backup_mode == BACKUP_MODE_FULL &&
98
		backup->status == BACKUP_STATUS_MERGING)
99
	{
100
		elog(WARNING, "Full backup %s has status %s, skip validation",
101
			backup_id_of(backup), status2str(backup->status));
102
		return;
103
	}
104

105
	if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE ||
106
		backup->status == BACKUP_STATUS_MERGING)
107
		elog(INFO, "Validating backup %s", backup_id_of(backup));
108
	else
109
		elog(INFO, "Revalidating backup %s", backup_id_of(backup));
110

111
	if (backup->backup_mode != BACKUP_MODE_FULL &&
112
		backup->backup_mode != BACKUP_MODE_DIFF_PAGE &&
113
		backup->backup_mode != BACKUP_MODE_DIFF_PTRACK &&
114
		backup->backup_mode != BACKUP_MODE_DIFF_DELTA)
115
		elog(WARNING, "Invalid backup_mode of backup %s", backup_id_of(backup));
116

117
	join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR);
118
	files = get_backup_filelist(backup, false);
119

120
	if (!files)
121
	{
122
		elog(WARNING, "Backup %s file list is corrupted", backup_id_of(backup));
123
		backup->status = BACKUP_STATUS_CORRUPT;
124
		write_backup_status(backup, BACKUP_STATUS_CORRUPT, true);
125
		return;
126
	}
127

128
//	if (params && params->partial_db_list)
129
//		dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list,
130
//														params->partial_restore_type);
131

132
	/* setup threads */
133
	pfilearray_clear_locks(files);
134

135
	/* init thread args with own file lists */
136
	threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads);
137
	threads_args = (validate_files_arg *)
138
		palloc(sizeof(validate_files_arg) * num_threads);
139

140
	/* Validate files */
141
	thread_interrupted = false;
142
	for (i = 0; i < num_threads; i++)
143
	{
144
		validate_files_arg *arg = &(threads_args[i]);
145

146
		arg->base_path = backup->database_dir;
147
		arg->files = files;
148
		arg->corrupted = false;
149
		arg->backup_mode = backup->backup_mode;
150
		arg->stop_lsn = backup->stop_lsn;
151
		arg->checksum_version = backup->checksum_version;
152
		arg->backup_version = parse_program_version(backup->program_version);
153
		arg->external_prefix = external_prefix;
154
		arg->hdr_map = &(backup->hdr_map);
155
//		arg->dbOid_exclude_list = dbOid_exclude_list;
156
		/* By default there are some error */
157
		threads_args[i].ret = 1;
158

159
		pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg);
160
	}
161

162
	/* Wait theads */
163
	for (i = 0; i < num_threads; i++)
164
	{
165
		validate_files_arg *arg = &(threads_args[i]);
166

167
		pthread_join(threads[i], NULL);
168
		if (arg->corrupted)
169
			corrupted = true;
170
		if (arg->ret == 1)
171
			validation_isok = false;
172
	}
173
	if (!validation_isok)
174
		elog(ERROR, "Data files validation failed");
175

176
	pfree(threads);
177
	pfree(threads_args);
178

179
	/* cleanup */
180
	parray_walk(files, pgFileFree);
181
	parray_free(files);
182
	cleanup_header_map(&(backup->hdr_map));
183

184
	/* Update backup status */
185
	if (corrupted)
186
		backup->status = BACKUP_STATUS_CORRUPT;
187

188
	write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT :
189
						BACKUP_STATUS_OK, true);
190

191
	if (corrupted)
192
		elog(WARNING, "Backup %s data files are corrupted", backup_id_of(backup));
193
	else
194
		elog(INFO, "Backup %s data files are valid", backup_id_of(backup));
195

196
	/* Issue #132 kludge */
197
	if (!corrupted &&
198
		((parse_program_version(backup->program_version) == 20104)||
199
		 (parse_program_version(backup->program_version) == 20105)||
200
		 (parse_program_version(backup->program_version) == 20201)))
201
	{
202
		char path[MAXPGPATH];
203

204
		join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
205

206
		if (pgFileSize(path) >= (BLCKSZ*500))
207
		{
208
			elog(WARNING, "Backup %s is a victim of metadata corruption. "
209
							"Additional information can be found here: "
210
							"https://github.com/postgrespro/pg_probackup/issues/132",
211
							backup_id_of(backup));
212
			backup->status = BACKUP_STATUS_CORRUPT;
213
			write_backup_status(backup, BACKUP_STATUS_CORRUPT, true);
214
		}
215
	}
216
}
217

218
/*
219
 * Validate files in the backup.
220
 * NOTE: If file is not valid, do not use ERROR log message,
221
 * rather throw a WARNING and set arguments->corrupted = true.
222
 * This is necessary to update backup status.
223
 */
224
static void *
225
pgBackupValidateFiles(void *arg)
226
{
227
	int			i;
228
	validate_files_arg *arguments = (validate_files_arg *)arg;
229
	int			num_files = parray_num(arguments->files);
230
	pg_crc32	crc;
231

232
	for (i = 0; i < num_files; i++)
233
	{
234
		struct stat st;
235
		pgFile	   *file = (pgFile *) parray_get(arguments->files, i);
236
		char        file_fullpath[MAXPGPATH];
237

238
		if (interrupted || thread_interrupted)
239
			elog(ERROR, "Interrupted during validate");
240

241
		/* Validate only regular files */
242
		if (!S_ISREG(file->mode))
243
			continue;
244

245
		/*
246
		 * If in partial validate, check if the file belongs to the database
247
		 * we exclude. Only files from pgdata can be skipped.
248
		 */
249
		//if (arguments->dbOid_exclude_list && file->external_dir_num == 0
250
		//	&& parray_bsearch(arguments->dbOid_exclude_list,
251
		//					   &file->dbOid, pgCompareOid))
252
		//{
253
		//	elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"",
254
		//		 file->rel_path);
255
		//	continue;
256
		//}
257

258
		if (!pg_atomic_test_set_flag(&file->lock))
259
			continue;
260

261
		if (progress)
262
			elog(INFO, "Progress: (%d/%d). Validate file \"%s\"",
263
				 i + 1, num_files, file->rel_path);
264

265
		/*
266
		 * Skip files which has no data, because they
267
		 * haven't changed between backups.
268
		 */
269
		if (file->write_size == BYTES_INVALID)
270
		{
271
			/* TODO: lookup corresponding merge bug */
272
			if (arguments->backup_mode == BACKUP_MODE_FULL)
273
			{
274
				/* It is illegal for file in FULL backup to have BYTES_INVALID */
275
				elog(WARNING, "Backup file \"%s\" has invalid size. Possible metadata corruption.",
276
					file->rel_path);
277
				arguments->corrupted = true;
278
				break;
279
			}
280
			else
281
				continue;
282
		}
283

284
		/* no point in trying to open empty file */
285
		if (file->write_size == 0)
286
			continue;
287

288
		if (file->external_dir_num)
289
		{
290
			char temp[MAXPGPATH];
291

292
			makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num);
293
			join_path_components(file_fullpath, temp, file->rel_path);
294
		}
295
		else
296
			join_path_components(file_fullpath, arguments->base_path, file->rel_path);
297

298
		/* TODO: it is redundant to check file existence using stat */
299
		if (stat(file_fullpath, &st) == -1)
300
		{
301
			if (errno == ENOENT)
302
				elog(WARNING, "Backup file \"%s\" is not found", file_fullpath);
303
			else
304
				elog(WARNING, "Cannot stat backup file \"%s\": %s",
305
					file_fullpath, strerror(errno));
306
			arguments->corrupted = true;
307
			break;
308
		}
309

310
		if (file->write_size != st.st_size)
311
		{
312
			elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu",
313
				 file_fullpath, (unsigned long) st.st_size, file->write_size);
314
			arguments->corrupted = true;
315
			break;
316
		}
317

318
		/*
319
		 * If option skip-block-validation is set, compute only file-level CRC for
320
		 * datafiles, otherwise check them block by block.
321
		 * Currently we don't compute checksums for
322
		 * cfs_compressed data files, so skip block validation for them.
323
		 */
324
		if (!file->is_datafile || skip_block_validation || file->is_cfs)
325
		{
326
			/*
327
			 * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we
328
			 * use CRC-32.
329
			 *
330
			 * pg_control stores its content and checksum of the content, calculated
331
			 * using CRC-32C. If we calculate checksum of the whole pg_control using
332
			 * CRC-32C we get same checksum constantly. It might be because of the
333
			 * CRC-32C algorithm.
334
			 * To avoid this problem we need to use different algorithm, CRC-32 in
335
			 * this case.
336
			 *
337
			 * Starting from 2.0.25 we calculate crc of pg_control differently.
338
			 */
339
			if (arguments->backup_version >= 20025 &&
340
				strcmp(file->name, "pg_control") == 0 &&
341
				!file->external_dir_num)
342
				crc = get_pgcontrol_checksum(arguments->base_path);
343
			else
344
				crc = pgFileGetCRC(file_fullpath,
345
								   arguments->backup_version <= 20021 ||
346
								   arguments->backup_version >= 20025,
347
								   false);
348
			if (crc != file->crc)
349
			{
350
				elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",
351
						file_fullpath, crc, file->crc);
352
				arguments->corrupted = true;
353
			}
354
		}
355
		else
356
		{
357
			/*
358
			 * validate relation block by block
359
			 * check page headers, checksums (if enabled)
360
			 * and compute checksum of the file
361
			 */
362
			if (!validate_file_pages(file, file_fullpath, arguments->stop_lsn,
363
								  arguments->checksum_version,
364
								  arguments->backup_version,
365
								  arguments->hdr_map))
366
				arguments->corrupted = true;
367
		}
368
	}
369

370
	/* Data files validation is successful */
371
	arguments->ret = 0;
372

373
	return NULL;
374
}
375

376
/*
377
 * Validate all backups in the backup catalog.
378
 * If --instance option was provided, validate only backups of this instance.
379
 *
380
 * TODO: split into two functions: do_validate_catalog and do_validate_instance.
381
 */
382
int
383
do_validate_all(CatalogState *catalogState, InstanceState *instanceState)
384
{
385
	corrupted_backup_found = false;
386
	skipped_due_to_lock = false;
387

388
	if (instanceState == NULL)
389
	{
390
		/* Show list of instances */
391
		DIR		   *dir;
392
		struct dirent *dent;
393

394
		/* open directory and list contents */
395
		dir = opendir(catalogState->backup_subdir_path);
396
		if (dir == NULL)
397
			elog(ERROR, "Cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno));
398

399
		errno = 0;
400
		while ((dent = readdir(dir)))
401
		{
402
			char		child[MAXPGPATH];
403
			struct stat	st;
404

405
			/* skip entries point current dir or parent dir */
406
			if (strcmp(dent->d_name, ".") == 0 ||
407
				strcmp(dent->d_name, "..") == 0)
408
				continue;
409

410
			join_path_components(child, catalogState->backup_subdir_path, dent->d_name);
411

412
			if (lstat(child, &st) == -1)
413
				elog(ERROR, "Cannot stat file \"%s\": %s", child, strerror(errno));
414

415
			if (!S_ISDIR(st.st_mode))
416
				continue;
417

418
			/*
419
			 * Initialize instance configuration.
420
			 */
421
			instanceState = pgut_new(InstanceState); /* memory leak */
422
			strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH);
423

424
			join_path_components(instanceState->instance_backup_subdir_path,
425
								catalogState->backup_subdir_path, instanceState->instance_name);
426
			join_path_components(instanceState->instance_wal_subdir_path,
427
								catalogState->wal_subdir_path, instanceState->instance_name);
428
			join_path_components(instanceState->instance_config_path,
429
								 instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE);
430

431
			if (config_read_opt(instanceState->instance_config_path, instance_options, ERROR, false,
432
								true) == 0)
433
			{
434
				elog(WARNING, "Configuration file \"%s\" is empty", instanceState->instance_config_path);
435
				corrupted_backup_found = true;
436
				continue;
437
			}
438

439
			do_validate_instance(instanceState);
440
		}
441
	}
442
	else
443
	{
444
		do_validate_instance(instanceState);
445
	}
446

447
	/* TODO: Probably we should have different exit code for every condition
448
	 * and they combination:
449
	 *  0 - all backups are valid
450
	 *  1 - some backups are corrupt
451
	 *  2 - some backups where skipped due to concurrent locks
452
	 *  3 - some backups are corrupt and some are skipped due to concurrent locks
453
	 */
454

455
	if (skipped_due_to_lock)
456
		elog(WARNING, "Some backups weren't locked and they were skipped");
457

458
	if (corrupted_backup_found)
459
	{
460
		elog(WARNING, "Some backups are not valid");
461
		return 1;
462
	}
463

464
	if (!skipped_due_to_lock && !corrupted_backup_found)
465
		elog(INFO, "All backups are valid");
466

467
	return 0;
468
}
469

470
/*
471
 * Validate all backups in the given instance of the backup catalog.
472
 */
473
static void
474
do_validate_instance(InstanceState *instanceState)
475
{
476
	int			i;
477
	int			j;
478
	parray	   *backups;
479
	pgBackup   *current_backup = NULL;
480

481
	elog(INFO, "Validate backups of the instance '%s'", instanceState->instance_name);
482

483
	/* Get list of all backups sorted in order of descending start time */
484
	backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID);
485

486
	/* Examine backups one by one and validate them */
487
	for (i = 0; i < parray_num(backups); i++)
488
	{
489
		pgBackup   *base_full_backup;
490

491
		current_backup = (pgBackup *) parray_get(backups, i);
492

493
		/* Find ancestor for incremental backup */
494
		if (current_backup->backup_mode != BACKUP_MODE_FULL)
495
		{
496
			pgBackup   *tmp_backup = NULL;
497
			int result;
498

499
			result = scan_parent_chain(current_backup, &tmp_backup);
500

501
			/* chain is broken */
502
			if (result == ChainIsBroken)
503
			{
504
				const char *parent_backup_id;
505
				const char *current_backup_id;
506
				/* determine missing backup ID */
507

508
				parent_backup_id = base36enc(tmp_backup->parent_backup);
509
				current_backup_id = backup_id_of(current_backup);
510
				corrupted_backup_found = true;
511

512
				/* orphanize current_backup */
513
				if (current_backup->status == BACKUP_STATUS_OK ||
514
					current_backup->status == BACKUP_STATUS_DONE)
515
				{
516
					write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true);
517
					elog(WARNING, "Backup %s is orphaned because his parent %s is missing",
518
							current_backup_id, parent_backup_id);
519
				}
520
				else
521
				{
522
					elog(WARNING, "Backup %s has missing parent %s",
523
						current_backup_id, parent_backup_id);
524
				}
525
				continue;
526
			}
527
			/* chain is whole, but at least one parent is invalid */
528
			else if (result == ChainIsInvalid)
529
			{
530
				/* Oldest corrupt backup has a chance for revalidation */
531
				if (current_backup->start_time != tmp_backup->start_time)
532
				{
533
					/* orphanize current_backup */
534
					if (current_backup->status == BACKUP_STATUS_OK ||
535
						current_backup->status == BACKUP_STATUS_DONE)
536
					{
537
						write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true);
538
						elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s",
539
								backup_id_of(current_backup),
540
								backup_id_of(tmp_backup),
541
								status2str(tmp_backup->status));
542
					}
543
					else
544
					{
545
						elog(WARNING, "Backup %s has parent %s with status: %s",
546
								backup_id_of(current_backup),
547
								backup_id_of(tmp_backup),
548
								status2str(tmp_backup->status));
549
					}
550
					continue;
551
				}
552
				base_full_backup = find_parent_full_backup(current_backup);
553

554
				/* sanity */
555
				if (!base_full_backup)
556
					elog(ERROR, "Parent full backup for the given backup %s was not found",
557
							backup_id_of(current_backup));
558
			}
559
			/* chain is whole, all parents are valid at first glance,
560
			 * current backup validation can proceed
561
			 */
562
			else
563
				base_full_backup = tmp_backup;
564
		}
565
		else
566
			base_full_backup = current_backup;
567

568
		/* Do not interrupt, validate the next backup */
569
		if (!lock_backup(current_backup, true, false))
570
		{
571
			elog(WARNING, "Cannot lock backup %s directory, skip validation",
572
				 backup_id_of(current_backup));
573
			skipped_due_to_lock = true;
574
			continue;
575
		}
576
		/* Valiate backup files*/
577
		pgBackupValidate(current_backup, NULL);
578

579
		/* Validate corresponding WAL files */
580
		if (current_backup->status == BACKUP_STATUS_OK)
581
			validate_wal(current_backup, instanceState->instance_wal_subdir_path, 0,
582
						 0, 0, current_backup->tli,
583
						 instance_config.xlog_seg_size);
584

585
		/*
586
		 * Mark every descendant of corrupted backup as orphan
587
		 */
588
		if (current_backup->status != BACKUP_STATUS_OK)
589
		{
590
			/* This is ridiculous but legal.
591
			 * PAGE_b2 <- OK
592
			 * PAGE_a2 <- OK
593
			 * PAGE_b1 <- ORPHAN
594
			 * PAGE_a1 <- CORRUPT
595
			 * FULL    <- OK
596
			 */
597

598
			corrupted_backup_found = true;
599

600
			for (j = i - 1; j >= 0; j--)
601
			{
602
				pgBackup   *backup = (pgBackup *) parray_get(backups, j);
603

604
				if (is_parent(current_backup->start_time, backup, false))
605
				{
606
					if (backup->status == BACKUP_STATUS_OK ||
607
						backup->status == BACKUP_STATUS_DONE)
608
					{
609
						write_backup_status(backup, BACKUP_STATUS_ORPHAN, true);
610

611
						elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s",
612
							 backup_id_of(backup),
613
							 backup_id_of(current_backup),
614
							 status2str(current_backup->status));
615
					}
616
				}
617
			}
618
		}
619

620
		/* For every OK backup we try to revalidate all his ORPHAN descendants. */
621
		if (current_backup->status == BACKUP_STATUS_OK)
622
		{
623
			/* revalidate all ORPHAN descendants
624
			 * be very careful not to miss a missing backup
625
			 * for every backup we must check that he is descendant of current_backup
626
			 */
627
			for (j = i - 1; j >= 0; j--)
628
			{
629
				pgBackup   *backup = (pgBackup *) parray_get(backups, j);
630
				pgBackup   *tmp_backup = NULL;
631
				int result;
632

633
				//PAGE_b2 ORPHAN
634
				//PAGE_b1 ORPHAN          -----
635
				//PAGE_a5 ORPHAN 			 |
636
				//PAGE_a4 CORRUPT 			 |
637
				//PAGE_a3 missing			 |
638
				//PAGE_a2 missing			 |
639
				//PAGE_a1 ORPHAN 			 |
640
				//PAGE    OK <- we are here<-|
641
				//FULL OK
642

643
				if (is_parent(current_backup->start_time, backup, false))
644
				{
645
					/* Revalidation make sense only if parent chain is whole.
646
					 * is_parent() do not guarantee that.
647
					 */
648
					result = scan_parent_chain(backup, &tmp_backup);
649

650
					if (result == ChainIsInvalid)
651
					{
652
						/* revalidation make sense only if oldest invalid backup is current_backup
653
						 */
654

655
						if (tmp_backup->start_time != backup->start_time)
656
							continue;
657

658
						if (backup->status == BACKUP_STATUS_ORPHAN)
659
						{
660
							/* Do not interrupt, validate the next backup */
661
							if (!lock_backup(backup, true, false))
662
							{
663
								elog(WARNING, "Cannot lock backup %s directory, skip validation",
664
									 backup_id_of(backup));
665
								skipped_due_to_lock = true;
666
								continue;
667
							}
668
							/* Revalidate backup files*/
669
							pgBackupValidate(backup, NULL);
670

671
							if (backup->status == BACKUP_STATUS_OK)
672
							{
673

674
								/* Revalidation successful, validate corresponding WAL files */
675
								validate_wal(backup, instanceState->instance_wal_subdir_path, 0,
676
											 0, 0, backup->tli,
677
											 instance_config.xlog_seg_size);
678
							}
679
						}
680

681
						if (backup->status != BACKUP_STATUS_OK)
682
						{
683
							corrupted_backup_found = true;
684
							continue;
685
						}
686
					}
687
				}
688
			}
689
		}
690
	}
691

692
	/* cleanup */
693
	parray_walk(backups, pgBackupFree);
694
	parray_free(backups);
695
}
696

697
/*
698
 * Validate tablespace_map checksum.
699
 * Error out in case of checksum mismatch.
700
 * Return 'false' if there are no tablespaces in backup.
701
 *
702
 * TODO: it is a bad, that we read the whole filelist just for
703
 * the sake of tablespace_map. Probably pgBackup should come with
704
 * already filled pgBackup.files
705
 */
706
bool
707
validate_tablespace_map(pgBackup *backup, bool no_validate)
708
{
709
	char        map_path[MAXPGPATH];
710
	pgFile     *dummy = NULL;
711
	pgFile    **tablespace_map = NULL;
712
	pg_crc32    crc;
713
	parray     *files = get_backup_filelist(backup, true);
714
	bool        use_crc32c = parse_program_version(backup->program_version) <= 20021 ||
715
                             parse_program_version(backup->program_version) >= 20025;
716

717
	parray_qsort(files, pgFileCompareRelPathWithExternal);
718
	join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE);
719

720
	dummy = pgFileInit(PG_TABLESPACE_MAP_FILE);
721
	tablespace_map = (pgFile **) parray_bsearch(files, dummy, pgFileCompareRelPathWithExternal);
722

723
	if (!tablespace_map)
724
	{
725
		elog(LOG, "there is no file tablespace_map");
726
		parray_walk(files, pgFileFree);
727
		parray_free(files);
728
		return false;
729
	}
730

731
	/* Exit if database/tablespace_map doesn't exist */
732
	if (!fileExists(map_path, FIO_BACKUP_HOST))
733
		elog(ERROR, "Tablespace map is missing: \"%s\", "
734
					"probably backup %s is corrupt, validate it",
735
			map_path, backup_id_of(backup));
736

737
	/* check tablespace map checksumms */
738
	if (!no_validate)
739
	{
740
		crc = pgFileGetCRC(map_path, use_crc32c, false);
741

742
		if ((*tablespace_map)->crc != crc)
743
			elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, "
744
						"probably backup %s is corrupt, validate it",
745
					map_path, crc, (*tablespace_map)->crc, backup_id_of(backup));
746
	}
747

748
	pgFileFree(dummy);
749
	parray_walk(files, pgFileFree);
750
	parray_free(files);
751
	return true;
752
}
753