pg_probackup

1
/*-------------------------------------------------------------------------
2
 *
3
 * dir.c: directory operation utility.
4
 *
5
 * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
6
 * Portions Copyright (c) 2015-2022, Postgres Professional
7
 *
8
 *-------------------------------------------------------------------------
9
 */
10

11
#include <assert.h>
12
#include "pg_probackup.h"
13
#include "utils/file.h"
14

15

16
#if PG_VERSION_NUM < 110000
17
#include "catalog/catalog.h"
18
#endif
19
#include "catalog/pg_tablespace.h"
20

21
#include <unistd.h>
22
#include <sys/stat.h>
23
#include <dirent.h>
24

25
#include "utils/configuration.h"
26

27
/*
28
 * The contents of these directories are removed or recreated during server
29
 * start so they are not included in backups.  The directories themselves are
30
 * kept and included as empty to preserve access permissions.
31
 */
32
static const char *pgdata_exclude_dir[] =
33
{
34
	PG_XLOG_DIR,
35
	/*
36
	 * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
37
	 * when stats_temp_directory is set because PGSS_TEXT_FILE is always created
38
	 * there.
39
	 */
40
	"pg_stat_tmp",
41
	"pgsql_tmp",
42

43
	/*
44
	 * It is generally not useful to backup the contents of this directory even
45
	 * if the intention is to restore to another master. See backup.sgml for a
46
	 * more detailed description.
47
	 */
48
	"pg_replslot",
49

50
	/* Contents removed on startup, see dsm_cleanup_for_mmap(). */
51
	"pg_dynshmem",
52

53
	/* Contents removed on startup, see AsyncShmemInit(). */
54
	"pg_notify",
55

56
	/*
57
	 * Old contents are loaded for possible debugging but are not required for
58
	 * normal operation, see OldSerXidInit().
59
	 */
60
	"pg_serial",
61

62
	/* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
63
	"pg_snapshots",
64

65
	/* Contents zeroed on startup, see StartupSUBTRANS(). */
66
	"pg_subtrans",
67

68
	/* end of list */
69
	NULL,				/* pg_log will be set later */
70
	NULL
71
};
72

73
static char *pgdata_exclude_files[] =
74
{
75
	/* Skip auto conf temporary file. */
76
	"postgresql.auto.conf.tmp",
77

78
	/* Skip current log file temporary file */
79
	"current_logfiles.tmp",
80
	"recovery.conf",
81
	"postmaster.pid",
82
	"postmaster.opts",
83
	"probackup_recovery.conf",
84
	"recovery.signal",
85
	"standby.signal",
86
	NULL
87
};
88

89
static char *pgdata_exclude_files_non_exclusive[] =
90
{
91
	/*skip in non-exclusive backup */
92
	"backup_label",
93
	"tablespace_map",
94
	NULL
95
};
96

97
/* Tablespace mapping structures */
98

99
typedef struct TablespaceListCell
100
{
101
	struct TablespaceListCell *next;
102
	char		old_dir[MAXPGPATH];
103
	char		new_dir[MAXPGPATH];
104
} TablespaceListCell;
105

106
typedef struct TablespaceList
107
{
108
	TablespaceListCell *head;
109
	TablespaceListCell *tail;
110
} TablespaceList;
111

112
typedef struct TablespaceCreatedListCell
113
{
114
	struct TablespaceCreatedListCell *next;
115
	char		link_name[MAXPGPATH];
116
	char		linked_dir[MAXPGPATH];
117
} TablespaceCreatedListCell;
118

119
typedef struct TablespaceCreatedList
120
{
121
	TablespaceCreatedListCell *head;
122
	TablespaceCreatedListCell *tail;
123
} TablespaceCreatedList;
124

125
static char dir_check_file(pgFile *file, bool backup_logs);
126

127
static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
128
								   bool exclude, bool follow_symlink, bool backup_logs,
129
								   bool skip_hidden, int external_dir_num, fio_location location);
130
static void opt_path_map(ConfigOption *opt, const char *arg,
131
						 TablespaceList *list, const char *type);
132
static void cleanup_tablespace(const char *path);
133

134
static void control_string_bad_format(const char* str);
135

136

137
/* Tablespace mapping */
138
static TablespaceList tablespace_dirs = {NULL, NULL};
139
/* Extra directories mapping */
140
static TablespaceList external_remap_list = {NULL, NULL};
141

142
/*
143
 * Create directory, also create parent directories if necessary.
144
 * In strict mode treat already existing directory as error.
145
 * Return values:
146
 *  0 - ok
147
 * -1 - error (check errno)
148
 */
149
int
150
dir_create_dir(const char *dir, mode_t mode, bool strict)
151
{
152
	char		parent[MAXPGPATH];
153

154
	strlcpy(parent, dir, MAXPGPATH);
155
	get_parent_directory(parent);
156

157
	/* Create parent first */
158
	if (strlen(parent) > 0 && access(parent, F_OK) == -1)
159
		dir_create_dir(parent, mode, false);
160

161
	/* Create directory */
162
	if (mkdir(dir, mode) == -1)
163
	{
164
		if (errno == EEXIST && !strict)	/* already exist */
165
			return 0;
166
		return -1;
167
	}
168

169
	return 0;
170
}
171

172
pgFile *
173
pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
174
		  int external_dir_num, fio_location location)
175
{
176
	struct stat		st;
177
	pgFile		   *file;
178

179
	/* stat the file */
180
	if (fio_stat(path, &st, follow_symlink, location) < 0)
181
	{
182
		/* file not found is not an error case */
183
		if (errno == ENOENT)
184
			return NULL;
185
		elog(ERROR, "Cannot stat file \"%s\": %s", path,
186
			strerror(errno));
187
	}
188

189
	file = pgFileInit(rel_path);
190
	file->size = st.st_size;
191
	file->mode = st.st_mode;
192
	file->mtime = st.st_mtime;
193
	file->external_dir_num = external_dir_num;
194

195
	return file;
196
}
197

198
pgFile *
199
pgFileInit(const char *rel_path)
200
{
201
	pgFile	   *file;
202
	char	   *file_name = NULL;
203

204
	file = (pgFile *) pgut_malloc(sizeof(pgFile));
205
	MemSet(file, 0, sizeof(pgFile));
206

207
	file->rel_path = pgut_strdup(rel_path);
208
	canonicalize_path(file->rel_path);
209

210
	/* Get file name from the path */
211
	file_name = last_dir_separator(file->rel_path);
212

213
	if (file_name == NULL)
214
		file->name = file->rel_path;
215
	else
216
	{
217
		file_name++;
218
		file->name = file_name;
219
	}
220

221
	/* Number of blocks readed during backup */
222
	file->n_blocks = BLOCKNUM_INVALID;
223

224
	/* Number of blocks backed up during backup */
225
	file->n_headers = 0;
226

227
	// May be add?
228
	// pg_atomic_clear_flag(file->lock);
229
	file->excluded = false;
230
	return file;
231
}
232

233
/*
234
 * Delete file pointed by the pgFile.
235
 * If the pgFile points directory, the directory must be empty.
236
 */
237
void
238
pgFileDelete(mode_t mode, const char *full_path)
239
{
240
	if (S_ISDIR(mode))
241
	{
242
		if (rmdir(full_path) == -1)
243
		{
244
			if (errno == ENOENT)
245
				return;
246
			else if (errno == ENOTDIR)	/* could be symbolic link */
247
				goto delete_file;
248

249
			elog(ERROR, "Cannot remove directory \"%s\": %s",
250
				full_path, strerror(errno));
251
		}
252
		return;
253
	}
254

255
delete_file:
256
	if (remove(full_path) == -1)
257
	{
258
		if (errno == ENOENT)
259
			return;
260
		elog(ERROR, "Cannot remove file \"%s\": %s", full_path,
261
			strerror(errno));
262
	}
263
}
264

265
void
266
pgFileFree(void *file)
267
{
268
	pgFile	   *file_ptr;
269

270
	if (file == NULL)
271
		return;
272

273
	file_ptr = (pgFile *) file;
274

275
	pfree(file_ptr->linked);
276
	pfree(file_ptr->rel_path);
277

278
	pfree(file);
279
}
280

281
/* Compare two pgFile with their path in ascending order of ASCII code. */
282
int
283
pgFileMapComparePath(const void *f1, const void *f2)
284
{
285
	page_map_entry *f1p = *(page_map_entry **)f1;
286
	page_map_entry *f2p = *(page_map_entry **)f2;
287

288
	return strcmp(f1p->path, f2p->path);
289
}
290

291
/* Compare two pgFile with their name in ascending order of ASCII code. */
292
int
293
pgFileCompareName(const void *f1, const void *f2)
294
{
295
	pgFile *f1p = *(pgFile **)f1;
296
	pgFile *f2p = *(pgFile **)f2;
297

298
	return strcmp(f1p->name, f2p->name);
299
}
300

301
/* Compare pgFile->name with string in ascending order of ASCII code. */
302
int
303
pgFileCompareNameWithString(const void *f1, const void *f2)
304
{
305
	pgFile *f1p = *(pgFile **)f1;
306
	char *f2s = *(char **)f2;
307

308
	return strcmp(f1p->name, f2s);
309
}
310

311
/* Compare pgFile->rel_path with string in ascending order of ASCII code. */
312
int
313
pgFileCompareRelPathWithString(const void *f1, const void *f2)
314
{
315
	pgFile *f1p = *(pgFile **)f1;
316
	char *f2s = *(char **)f2;
317

318
	return strcmp(f1p->rel_path, f2s);
319
}
320

321
/*
322
 * Compare two pgFile with their relative path and external_dir_num in ascending
323
 * order of ASСII code.
324
 */
325
int
326
pgFileCompareRelPathWithExternal(const void *f1, const void *f2)
327
{
328
	pgFile *f1p = *(pgFile **)f1;
329
	pgFile *f2p = *(pgFile **)f2;
330
	int 		res;
331

332
	res = strcmp(f1p->rel_path, f2p->rel_path);
333
	if (res == 0)
334
	{
335
		if (f1p->external_dir_num > f2p->external_dir_num)
336
			return 1;
337
		else if (f1p->external_dir_num < f2p->external_dir_num)
338
			return -1;
339
		else
340
			return 0;
341
	}
342
	return res;
343
}
344

345
/*
346
 * Compare two pgFile with their rel_path and external_dir_num
347
 * in descending order of ASCII code.
348
 */
349
int
350
pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2)
351
{
352
	return -pgFileCompareRelPathWithExternal(f1, f2);
353
}
354

355
/* Compare two pgFile with their linked directory path. */
356
int
357
pgFileCompareLinked(const void *f1, const void *f2)
358
{
359
	pgFile	   *f1p = *(pgFile **)f1;
360
	pgFile	   *f2p = *(pgFile **)f2;
361

362
	return strcmp(f1p->linked, f2p->linked);
363
}
364

365
/* Compare two pgFile with their size */
366
int
367
pgFileCompareSize(const void *f1, const void *f2)
368
{
369
	pgFile *f1p = *(pgFile **)f1;
370
	pgFile *f2p = *(pgFile **)f2;
371

372
	if (f1p->size > f2p->size)
373
		return 1;
374
	else if (f1p->size < f2p->size)
375
		return -1;
376
	else
377
		return 0;
378
}
379

380
/* Compare two pgFile with their size in descending order */
381
int
382
pgFileCompareSizeDesc(const void *f1, const void *f2)
383
{
384
	return -1 * pgFileCompareSize(f1, f2);
385
}
386

387
int
388
pgCompareString(const void *str1, const void *str2)
389
{
390
	return strcmp(*(char **) str1, *(char **) str2);
391
}
392

393
/*
394
 * From bsearch(3): "The compar routine is expected to have two argu‐
395
 * ments  which  point  to  the key object and to an array member, in that order"
396
 * But in practice this is opposite, so we took strlen from second string (search key)
397
 * This is checked by tests.catchup.CatchupTest.test_catchup_with_exclude_path
398
 */
399
int
400
pgPrefixCompareString(const void *str1, const void *str2)
401
{
402
	const char *s1 = *(char **) str1;
403
	const char *s2 = *(char **) str2;
404
	return strncmp(s1, s2, strlen(s2));
405
}
406

407
/* Compare two Oids */
408
int
409
pgCompareOid(const void *f1, const void *f2)
410
{
411
	Oid *v1 = *(Oid **) f1;
412
	Oid *v2 = *(Oid **) f2;
413

414
	if (*v1 > *v2)
415
		return 1;
416
	else if (*v1 < *v2)
417
		return -1;
418
	else
419
		return 0;}
420

421

422
void
423
db_map_entry_free(void *entry)
424
{
425
	db_map_entry *m = (db_map_entry *) entry;
426

427
	free(m->datname);
428
	free(entry);
429
}
430

431
/*
432
 * List files, symbolic links and directories in the directory "root" and add
433
 * pgFile objects to "files".  We add "root" to "files" if add_root is true.
434
 *
435
 * When follow_symlink is true, symbolic link is ignored and only file or
436
 * directory linked to will be listed.
437
 *
438
 * TODO: make it strictly local
439
 */
440
void
441
dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
442
			  bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num,
443
			  fio_location location)
444
{
445
	pgFile	   *file;
446

447
	file = pgFileNew(root, "", follow_symlink, external_dir_num, location);
448
	if (file == NULL)
449
	{
450
		/* For external directory this is not ok */
451
		if (external_dir_num > 0)
452
			elog(ERROR, "External directory is not found: \"%s\"", root);
453
		else
454
			return;
455
	}
456

457
	if (!S_ISDIR(file->mode))
458
	{
459
		if (external_dir_num > 0)
460
			elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected",
461
					root);
462
		else
463
			elog(WARNING, "Skip \"%s\": unexpected file format", root);
464
		return;
465
	}
466
	if (add_root)
467
		parray_append(files, file);
468

469
	dir_list_file_internal(files, file, root, exclude, follow_symlink,
470
						   backup_logs, skip_hidden, external_dir_num, location);
471

472
	if (!add_root)
473
		pgFileFree(file);
474
}
475

476
#define CHECK_FALSE				0
477
#define CHECK_TRUE				1
478
#define CHECK_EXCLUDE_FALSE		2
479

480
/*
481
 * Check file or directory.
482
 *
483
 * Check for exclude.
484
 * Extract information about the file parsing its name.
485
 * Skip files:
486
 * - skip temp tables files
487
 * - skip unlogged tables files
488
 * Skip recursive tablespace content
489
 * Set flags for:
490
 * - database directories
491
 * - datafiles
492
 */
493
static char
494
dir_check_file(pgFile *file, bool backup_logs)
495
{
496
	int			i;
497
	int			sscanf_res;
498
	bool		in_tablespace = false;
499

500
	in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path);
501

502
	/* Check if we need to exclude file by name */
503
	if (S_ISREG(file->mode))
504
	{
505
		if (!exclusive_backup)
506
		{
507
			for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++)
508
				if (strcmp(file->rel_path,
509
						   pgdata_exclude_files_non_exclusive[i]) == 0)
510
				{
511
					/* Skip */
512
					elog(LOG, "Excluding file: %s", file->name);
513
					return CHECK_FALSE;
514
				}
515
		}
516

517
		for (i = 0; pgdata_exclude_files[i]; i++)
518
			if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0)
519
			{
520
				/* Skip */
521
				elog(LOG, "Excluding file: %s", file->name);
522
				return CHECK_FALSE;
523
			}
524
	}
525
	/*
526
	 * If the directory name is in the exclude list, do not list the
527
	 * contents.
528
	 */
529
	else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0)
530
	{
531
		/*
532
		 * If the item in the exclude list starts with '/', compare to
533
		 * the absolute path of the directory. Otherwise compare to the
534
		 * directory name portion.
535
		 */
536
		for (i = 0; pgdata_exclude_dir[i]; i++)
537
		{
538
			/* exclude by dirname */
539
			if (strcmp(file->name, pgdata_exclude_dir[i]) == 0)
540
			{
541
				elog(LOG, "Excluding directory content: %s", file->rel_path);
542
				return CHECK_EXCLUDE_FALSE;
543
			}
544
		}
545

546
		if (!backup_logs)
547
		{
548
			if (strcmp(file->rel_path, PG_LOG_DIR) == 0)
549
			{
550
				/* Skip */
551
				elog(LOG, "Excluding directory content: %s", file->rel_path);
552
				return CHECK_EXCLUDE_FALSE;
553
			}
554
		}
555
	}
556

557
	/*
558
	 * Do not copy tablespaces twice. It may happen if the tablespace is located
559
	 * inside the PGDATA.
560
	 */
561
	if (S_ISDIR(file->mode) &&
562
		strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0)
563
	{
564
		Oid			tblspcOid;
565
		char		tmp_rel_path[MAXPGPATH];
566

567
		/*
568
		 * Valid path for the tablespace is
569
		 * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
570
		 */
571
		if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
572
			return CHECK_FALSE;
573
		sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s",
574
							&tblspcOid, tmp_rel_path);
575
		if (sscanf_res == 0)
576
			return CHECK_FALSE;
577
	}
578

579
	if (in_tablespace)
580
	{
581
		char		tmp_rel_path[MAXPGPATH];
582

583
		sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/",
584
							&(file->tblspcOid), tmp_rel_path,
585
							&(file->dbOid));
586

587
		/*
588
		 * We should skip other files and directories rather than
589
		 * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace.
590
		 */
591
		if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0)
592
			return CHECK_FALSE;
593
	}
594
	else if (path_is_prefix_of_path("global", file->rel_path))
595
	{
596
		file->tblspcOid = GLOBALTABLESPACE_OID;
597
	}
598
	else if (path_is_prefix_of_path("base", file->rel_path))
599
	{
600
		file->tblspcOid = DEFAULTTABLESPACE_OID;
601

602
		sscanf(file->rel_path, "base/%u/", &(file->dbOid));
603
	}
604

605
	/* Do not backup ptrack_init files */
606
	if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0)
607
		return CHECK_FALSE;
608

609
	/*
610
	 * Check files located inside database directories including directory
611
	 * 'global'
612
	 */
613
	if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
614
		file->name && file->name[0])
615
	{
616
		if (strcmp(file->name, "pg_internal.init") == 0)
617
			return CHECK_FALSE;
618
		/* Do not backup ptrack2.x temp map files */
619
//		else if (strcmp(file->name, "ptrack.map") == 0)
620
//			return CHECK_FALSE;
621
		else if (strcmp(file->name, "ptrack.map.mmap") == 0)
622
			return CHECK_FALSE;
623
		else if (strcmp(file->name, "ptrack.map.tmp") == 0)
624
			return CHECK_FALSE;
625
		/* Do not backup temp files */
626
		else if (file->name[0] == 't' && isdigit(file->name[1]))
627
			return CHECK_FALSE;
628
		else if (isdigit(file->name[0]))
629
		{
630
			set_forkname(file);
631

632
			if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */
633
				return CHECK_FALSE;
634
		}
635
	}
636

637
	return CHECK_TRUE;
638
}
639

640
/*
641
 * List files in parent->path directory.  If "exclude" is true do not add into
642
 * "files" files from pgdata_exclude_files and directories from
643
 * pgdata_exclude_dir.
644
 *
645
 * TODO: should we check for interrupt here ?
646
 */
647
static void
648
dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
649
					   bool exclude, bool follow_symlink, bool backup_logs,
650
					   bool skip_hidden, int external_dir_num, fio_location location)
651
{
652
	DIR			  *dir;
653
	struct dirent *dent;
654

655
	if (!S_ISDIR(parent->mode))
656
		elog(ERROR, "\"%s\" is not a directory", parent_dir);
657

658
	/* Open directory and list contents */
659
	dir = fio_opendir(parent_dir, location);
660
	if (dir == NULL)
661
	{
662
		if (errno == ENOENT)
663
		{
664
			/* Maybe the directory was removed */
665
			return;
666
		}
667
		elog(ERROR, "Cannot open directory \"%s\": %s",
668
				parent_dir, strerror(errno));
669
	}
670

671
	errno = 0;
672
	while ((dent = fio_readdir(dir)))
673
	{
674
		pgFile	   *file;
675
		char		child[MAXPGPATH];
676
		char		rel_child[MAXPGPATH];
677
		char		check_res;
678

679
		join_path_components(child, parent_dir, dent->d_name);
680
		join_path_components(rel_child, parent->rel_path, dent->d_name);
681

682
		file = pgFileNew(child, rel_child, follow_symlink, external_dir_num,
683
						 location);
684
		if (file == NULL)
685
			continue;
686

687
		/* Skip entries point current dir or parent dir */
688
		if (S_ISDIR(file->mode) &&
689
			(strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0))
690
		{
691
			pgFileFree(file);
692
			continue;
693
		}
694

695
		/* skip hidden files and directories */
696
		if (skip_hidden && file->name[0] == '.')
697
		{
698
			elog(WARNING, "Skip hidden file: '%s'", child);
699
			pgFileFree(file);
700
			continue;
701
		}
702

703
		/*
704
		 * Add only files, directories and links. Skip sockets and other
705
		 * unexpected file formats.
706
		 */
707
		if (!S_ISDIR(file->mode) && !S_ISREG(file->mode))
708
		{
709
			elog(WARNING, "Skip '%s': unexpected file format", child);
710
			pgFileFree(file);
711
			continue;
712
		}
713

714
		if (exclude)
715
		{
716
			check_res = dir_check_file(file, backup_logs);
717
			if (check_res == CHECK_FALSE)
718
			{
719
				/* Skip */
720
				pgFileFree(file);
721
				continue;
722
			}
723
			else if (check_res == CHECK_EXCLUDE_FALSE)
724
			{
725
				/* We add the directory itself which content was excluded */
726
				parray_append(files, file);
727
				continue;
728
			}
729
		}
730

731
		parray_append(files, file);
732

733
		/*
734
		 * If the entry is a directory call dir_list_file_internal()
735
		 * recursively.
736
		 */
737
		if (S_ISDIR(file->mode))
738
			dir_list_file_internal(files, file, child, exclude, follow_symlink,
739
								   backup_logs, skip_hidden, external_dir_num, location);
740
	}
741

742
	if (errno && errno != ENOENT)
743
	{
744
		int			errno_tmp = errno;
745
		fio_closedir(dir);
746
		elog(ERROR, "Cannot read directory \"%s\": %s",
747
				parent_dir, strerror(errno_tmp));
748
	}
749
	fio_closedir(dir);
750
}
751

752
/*
753
 * Retrieve tablespace path, either relocated or original depending on whether
754
 * -T was passed or not.
755
 *
756
 * Copy of function get_tablespace_mapping() from pg_basebackup.c.
757
 */
758
const char *
759
get_tablespace_mapping(const char *dir)
760
{
761
	TablespaceListCell *cell;
762

763
	for (cell = tablespace_dirs.head; cell; cell = cell->next)
764
		if (strcmp(dir, cell->old_dir) == 0)
765
			return cell->new_dir;
766

767
	return dir;
768
}
769

770
/*
771
 * Split argument into old_dir and new_dir and append to mapping
772
 * list.
773
 *
774
 * Copy of function tablespace_list_append() from pg_basebackup.c.
775
 */
776
static void
777
opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list,
778
			 const char *type)
779
{
780
	TablespaceListCell *cell = pgut_new(TablespaceListCell);
781
	char	   *dst;
782
	char	   *dst_ptr;
783
	const char *arg_ptr;
784

785
	memset(cell, 0, sizeof(TablespaceListCell));
786
	dst_ptr = dst = cell->old_dir;
787
	for (arg_ptr = arg; *arg_ptr; arg_ptr++)
788
	{
789
		if (dst_ptr - dst >= MAXPGPATH)
790
			elog(ERROR, "Directory name too long");
791

792
		if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
793
			;					/* skip backslash escaping = */
794
		else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
795
		{
796
			if (*cell->new_dir)
797
				elog(ERROR, "Multiple \"=\" signs in %s mapping\n", type);
798
			else
799
				dst = dst_ptr = cell->new_dir;
800
		}
801
		else
802
			*dst_ptr++ = *arg_ptr;
803
	}
804

805
	if (!*cell->old_dir || !*cell->new_dir)
806
		elog(ERROR, "Invalid %s mapping format \"%s\", "
807
			 "must be \"OLDDIR=NEWDIR\"", type, arg);
808
	canonicalize_path(cell->old_dir);
809
	canonicalize_path(cell->new_dir);
810

811
	/*
812
	 * This check isn't absolutely necessary.  But all tablespaces are created
813
	 * with absolute directories, so specifying a non-absolute path here would
814
	 * just never match, possibly confusing users.  It's also good to be
815
	 * consistent with the new_dir check.
816
	 */
817
	if (!is_absolute_path(cell->old_dir))
818
		elog(ERROR, "Old directory is not an absolute path in %s mapping: %s\n",
819
			 type, cell->old_dir);
820

821
	if (!is_absolute_path(cell->new_dir))
822
		elog(ERROR, "New directory is not an absolute path in %s mapping: %s\n",
823
			 type, cell->new_dir);
824

825
	if (list->tail)
826
		list->tail->next = cell;
827
	else
828
		list->head = cell;
829
	list->tail = cell;
830
}
831

832
/* Parse tablespace mapping */
833
void
834
opt_tablespace_map(ConfigOption *opt, const char *arg)
835
{
836
	opt_path_map(opt, arg, &tablespace_dirs, "tablespace");
837
}
838

839
/* Parse external directories mapping */
840
void
841
opt_externaldir_map(ConfigOption *opt, const char *arg)
842
{
843
	opt_path_map(opt, arg, &external_remap_list, "external directory");
844
}
845

846
/*
847
 * Create directories from **dest_files** in **data_dir**.
848
 *
849
 * If **extract_tablespaces** is true then try to extract tablespace data
850
 * directories into their initial path using tablespace_map file.
851
 * Use **backup_dir** for tablespace_map extracting.
852
 *
853
 * Enforce permissions from backup_content.control. The only
854
 * problem now is with PGDATA itself.
855
 * TODO: we must preserve PGDATA permissions somewhere. Is it actually a problem?
856
 * Shouldn`t starting postgres force correct permissions on PGDATA?
857
 *
858
 * TODO: symlink handling. If user located symlink in PG_TBLSPC_DIR, it will
859
 * be restored as directory.
860
 */
861
void
862
create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir,
863
						bool extract_tablespaces, bool incremental, fio_location location, 
864
						const char* waldir_path)
865
{
866
	int			i;
867
	parray		*links = NULL;
868
	mode_t		pg_tablespace_mode = DIR_PERMISSION;
869
	char		to_path[MAXPGPATH];
870

871
	if (waldir_path && !dir_is_empty(waldir_path, location))
872
	{
873
		elog(ERROR, "WAL directory location is not empty: \"%s\"", waldir_path);
874
	}
875

876

877
	/* get tablespace map */
878
	if (extract_tablespaces)
879
	{
880
		links = parray_new();
881
		read_tablespace_map(links, backup_dir);
882
		/* Sort links by a link name */
883
		parray_qsort(links, pgFileCompareName);
884
	}
885

886
	/*
887
	 * We have no idea about tablespace permission
888
	 * For PG < 11 we can just force default permissions.
889
	 */
890
#if PG_VERSION_NUM >= 110000
891
	if (links)
892
	{
893
		/* For PG>=11 we use temp kludge: trust permissions on 'pg_tblspc'
894
		 * and force them on every tablespace.
895
		 * TODO: remove kludge and ask data_directory_mode
896
		 * at the start of backup.
897
		 */
898
		for (i = 0; i < parray_num(dest_files); i++)
899
		{
900
			pgFile	   *file = (pgFile *) parray_get(dest_files, i);
901

902
			if (!S_ISDIR(file->mode))
903
				continue;
904

905
			/* skip external directory content */
906
			if (file->external_dir_num != 0)
907
				continue;
908

909
			/* look for 'pg_tblspc' directory  */
910
			if (strcmp(file->rel_path, PG_TBLSPC_DIR) == 0)
911
			{
912
				pg_tablespace_mode = file->mode;
913
				break;
914
			}
915
		}
916
	}
917
#endif
918

919
	/*
920
	 * We iterate over dest_files and for every directory with parent 'pg_tblspc'
921
	 * we must lookup this directory name in tablespace map.
922
	 * If we got a match, we treat this directory as tablespace.
923
	 * It means that we create directory specified in tablespace_map and
924
	 * original directory created as symlink to it.
925
	 */
926

927
	elog(LOG, "Restore directories and symlinks...");
928

929
	/* create directories */
930
	for (i = 0; i < parray_num(dest_files); i++)
931
	{
932
		char parent_dir[MAXPGPATH];
933
		pgFile	   *dir = (pgFile *) parray_get(dest_files, i);
934

935
		if (!S_ISDIR(dir->mode))
936
			continue;
937

938
		/* skip external directory content */
939
		if (dir->external_dir_num != 0)
940
			continue;
941
		/* Create WAL directory and symlink if waldir_path is setting */
942
		if (waldir_path && strcmp(dir->rel_path, PG_XLOG_DIR) == 0) {
943
			/* get full path to PG_XLOG_DIR */
944

945
			join_path_components(to_path, data_dir, PG_XLOG_DIR);
946

947
			elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"",
948
				waldir_path, to_path);
949

950
			/* create tablespace directory from waldir_path*/
951
			fio_mkdir(waldir_path, pg_tablespace_mode, location);
952

953
			/* create link to linked_path */
954
			if (fio_symlink(waldir_path, to_path, incremental, location) < 0)
955
				elog(ERROR, "Could not create symbolic link \"%s\": %s",
956
					to_path, strerror(errno));
957

958
			continue;
959

960

961
		}
962

963
		/* tablespace_map exists */
964
		if (links)
965
		{
966
			/* get parent dir of rel_path */
967
			strlcpy(parent_dir, dir->rel_path, MAXPGPATH);
968
			get_parent_directory(parent_dir);
969

970
			/* check if directory is actually link to tablespace */
971
			if (strcmp(parent_dir, PG_TBLSPC_DIR) == 0)
972
			{
973
				/* this directory located in pg_tblspc
974
				 * check it against tablespace map
975
				 */
976
				pgFile **link = (pgFile **) parray_bsearch(links, dir, pgFileCompareName);
977

978
				/* got match */
979
				if (link)
980
				{
981
					const char *linked_path = get_tablespace_mapping((*link)->linked);
982

983
					if (!is_absolute_path(linked_path))
984
							elog(ERROR, "Tablespace directory path must be an absolute path: %s\n",
985
								 linked_path);
986

987
					join_path_components(to_path, data_dir, dir->rel_path);
988

989
					elog(LOG, "Create directory \"%s\" and symbolic link \"%s\"",
990
							 linked_path, to_path);
991

992
					/* create tablespace directory */
993
					fio_mkdir(linked_path, pg_tablespace_mode, location);
994

995
					/* create link to linked_path */
996
					if (fio_symlink(linked_path, to_path, incremental, location) < 0)
997
						elog(ERROR, "Could not create symbolic link \"%s\": %s",
998
							 to_path, strerror(errno));
999

1000
					continue;
1001
				}
1002
			}
1003
		}
1004

1005
		/* This is not symlink, create directory */
1006
		elog(LOG, "Create directory \"%s\"", dir->rel_path);
1007

1008
		join_path_components(to_path, data_dir, dir->rel_path);
1009

1010
		// TODO check exit code
1011
		fio_mkdir(to_path, dir->mode, location);
1012
	}
1013

1014
	if (extract_tablespaces)
1015
	{
1016
		parray_walk(links, pgFileFree);
1017
		parray_free(links);
1018
	}
1019
}
1020

1021
/*
1022
 * Read names of symbolic names of tablespaces with links to directories from
1023
 * tablespace_map or tablespace_map.txt.
1024
 */
1025
void
1026
read_tablespace_map(parray *links, const char *backup_dir)
1027
{
1028
	FILE	   *fp;
1029
	char		db_path[MAXPGPATH],
1030
				map_path[MAXPGPATH];
1031
	char		buf[MAXPGPATH * 2];
1032

1033
	join_path_components(db_path, backup_dir, DATABASE_DIR);
1034
	join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE);
1035

1036
	fp = fio_open_stream(map_path, FIO_BACKUP_HOST);
1037
	if (fp == NULL)
1038
		elog(ERROR, "Cannot open tablespace map file \"%s\": %s", map_path, strerror(errno));
1039

1040
	while (fgets(buf, lengthof(buf), fp))
1041
	{
1042
		char        link_name[MAXPGPATH];
1043
		char       *path;
1044
		int         n = 0;
1045
		pgFile     *file;
1046
		int         i = 0;
1047

1048
		if (sscanf(buf, "%s %n", link_name, &n) != 1)
1049
			elog(ERROR, "Invalid format found in \"%s\"", map_path);
1050

1051
		path = buf + n;
1052

1053
		/* Remove newline character at the end of string if any  */
1054
		i = strcspn(path, "\n");
1055
		if (strlen(path) > i)
1056
			path[i] = '\0';
1057

1058
		file = pgut_new(pgFile);
1059
		memset(file, 0, sizeof(pgFile));
1060

1061
		/* follow the convention for pgFileFree */
1062
		file->name = pgut_strdup(link_name);
1063
		file->linked = pgut_strdup(path);
1064
		canonicalize_path(file->linked);
1065

1066
		parray_append(links, file);
1067
	}
1068

1069
	if (ferror(fp))
1070
			elog(ERROR, "Failed to read from file: \"%s\"", map_path);
1071

1072
	fio_close_stream(fp);
1073
}
1074

1075
/*
1076
 * Check that all tablespace mapping entries have correct linked directory
1077
 * paths. Linked directories must be empty or do not exist, unless
1078
 * we are running incremental restore, then linked directories can be nonempty.
1079
 *
1080
 * If tablespace-mapping option is supplied, all OLDDIR entries must have
1081
 * entries in tablespace_map file.
1082
 *
1083
 * When running incremental restore with tablespace remapping, then
1084
 * new tablespace directory MUST be empty, because there is no way
1085
 * we can be sure, that files laying there belong to our instance.
1086
 * But "force" flag allows to ignore this condition, by wiping out
1087
 * the current content on the directory.
1088
 *
1089
 * Exit codes:
1090
 *  1. backup has no tablespaces
1091
 *  2. backup has tablespaces and they are empty
1092
 *  3. backup has tablespaces and some of them are not empty
1093
 */
1094
int
1095
check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate)
1096
{
1097
	parray	   *links = parray_new();
1098
	size_t		i;
1099
	TablespaceListCell *cell;
1100
	pgFile	   *tmp_file = pgut_new(pgFile);
1101
	bool        tblspaces_are_empty = true;
1102

1103
	elog(LOG, "Checking tablespace directories of backup %s",
1104
			backup_id_of(backup));
1105

1106
	/* validate tablespace map,
1107
	 * if there are no tablespaces, then there is nothing left to do
1108
	 */
1109
	if (!validate_tablespace_map(backup, no_validate))
1110
	{
1111
		/*
1112
		 * Sanity check
1113
		 * If there is no tablespaces in backup,
1114
		 * then using the '--tablespace-mapping' option is a mistake.
1115
		 */
1116
		if (tablespace_dirs.head != NULL)
1117
			elog(ERROR, "Backup %s has no tablespaceses, nothing to remap "
1118
					"via \"--tablespace-mapping\" option", backup_id_of(backup));
1119
		return NoTblspc;
1120
	}
1121

1122
	read_tablespace_map(links, backup->root_dir);
1123
	/* Sort links by the path of a linked file*/
1124
	parray_qsort(links, pgFileCompareLinked);
1125

1126
	/* 1 - each OLDDIR must have an entry in tablespace_map file (links) */
1127
	for (cell = tablespace_dirs.head; cell; cell = cell->next)
1128
	{
1129
		tmp_file->linked = cell->old_dir;
1130

1131
		if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL)
1132
			elog(ERROR, "--tablespace-mapping option's old directory "
1133
				 "doesn't have an entry in tablespace_map file: \"%s\"",
1134
				 cell->old_dir);
1135
	}
1136

1137
	/*
1138
	 * There is difference between incremental restore of already existing
1139
	 * tablespaceses and remapped tablespaceses.
1140
	 * Former are allowed to be not empty, because we treat them like an
1141
	 * extension of PGDATA.
1142
	 * The latter are not, unless "--force" flag is used.
1143
	 * in which case the remapped directory is nuked - just to be safe,
1144
	 * because it is hard to be sure that there are no some tricky corner
1145
	 * cases of pages from different systems having the same crc.
1146
	 * This is a strict approach.
1147
	 *
1148
	 * Why can`t we not nuke it and just let it roll ?
1149
	 * What if user just wants to rerun failed restore with the same
1150
	 * parameters? Nuking is bad for this case.
1151
	 *
1152
	 * Consider the example of existing PGDATA:
1153
	 * ....
1154
	 * 	pg_tablespace
1155
	 * 		100500-> /somedirectory
1156
	 * ....
1157
	 *
1158
	 * We want to remap it during restore like that:
1159
	 * ....
1160
	 * 	pg_tablespace
1161
	 * 		100500-> /somedirectory1
1162
	 * ....
1163
	 *
1164
	 * Usually it is required for "/somedirectory1" to be empty, but
1165
	 * in case of incremental restore with 'force' flag, which required
1166
	 * of us to drop already existing content of "/somedirectory1".
1167
	 *
1168
	 * TODO: Ideally in case of incremental restore we must also
1169
	 * drop the "/somedirectory" directory first, but currently
1170
	 * we don`t do that.
1171
	 */
1172

1173
	/* 2 - all linked directories must be empty */
1174
	for (i = 0; i < parray_num(links); i++)
1175
	{
1176
		pgFile	   *link = (pgFile *) parray_get(links, i);
1177
		const char *linked_path = link->linked;
1178
		bool remapped = false;
1179

1180
		for (cell = tablespace_dirs.head; cell; cell = cell->next)
1181
		{
1182
			if (strcmp(link->linked, cell->old_dir) == 0)
1183
			{
1184
				linked_path = cell->new_dir;
1185
				remapped = true;
1186
				break;
1187
			}
1188
		}
1189

1190
		if (remapped)
1191
			elog(INFO, "Tablespace %s will be remapped from \"%s\" to \"%s\"", link->name, cell->old_dir, cell->new_dir);
1192
		else
1193
			elog(INFO, "Tablespace %s will be restored using old path \"%s\"", link->name, linked_path);
1194

1195
		if (!is_absolute_path(linked_path))
1196
			elog(ERROR, "Tablespace directory path must be an absolute path: %s\n",
1197
				 linked_path);
1198

1199
		if (!dir_is_empty(linked_path, FIO_DB_HOST))
1200
		{
1201

1202
			if (!incremental)
1203
				elog(ERROR, "Restore tablespace destination is not empty: \"%s\"", linked_path);
1204

1205
			else if (remapped && !force)
1206
				elog(ERROR, "Remapped tablespace destination is not empty: \"%s\". "
1207
							"Use \"--force\" flag if you want to automatically clean up the "
1208
							"content of new tablespace destination",
1209
						linked_path);
1210

1211
			else if (pgdata_is_empty && !force)
1212
				elog(ERROR, "PGDATA is empty, but tablespace destination is not: \"%s\". "
1213
							"Use \"--force\" flag is you want to automatically clean up the "
1214
							"content of tablespace destination",
1215
						linked_path);
1216

1217
			/*
1218
			 * TODO: compile the list of tblspc Oids to delete later,
1219
			 * similar to what we do with database_map.
1220
			 */
1221
			else if (force && (pgdata_is_empty || remapped))
1222
			{
1223
				elog(WARNING, "Cleaning up the content of %s directory: \"%s\"",
1224
						remapped ? "remapped tablespace" : "tablespace", linked_path);
1225
				cleanup_tablespace(linked_path);
1226
				continue;
1227
			}
1228

1229
			tblspaces_are_empty = false;
1230
		}
1231
	}
1232

1233
	free(tmp_file);
1234
	parray_walk(links, pgFileFree);
1235
	parray_free(links);
1236

1237
	if (tblspaces_are_empty)
1238
		return EmptyTblspc;
1239

1240
	return NotEmptyTblspc;
1241
}
1242

1243
/* TODO: Make it consistent with check_tablespace_mapping */
1244
void
1245
check_external_dir_mapping(pgBackup *backup, bool incremental)
1246
{
1247
	TablespaceListCell *cell;
1248
	parray *external_dirs_to_restore;
1249
	int		i;
1250

1251
	elog(LOG, "check external directories of backup %s",
1252
			backup_id_of(backup));
1253

1254
	if (!backup->external_dir_str)
1255
	{
1256
	 	if (external_remap_list.head)
1257
			elog(ERROR, "--external-mapping option's old directory doesn't "
1258
				 "have an entry in list of external directories of current "
1259
				 "backup: \"%s\"", external_remap_list.head->old_dir);
1260
		return;
1261
	}
1262

1263
	external_dirs_to_restore = make_external_directory_list(
1264
													backup->external_dir_str,
1265
													false);
1266
	/* 1 - each OLDDIR must have an entry in external_dirs_to_restore */
1267
	for (cell = external_remap_list.head; cell; cell = cell->next)
1268
	{
1269
		bool		found = false;
1270

1271
		for (i = 0; i < parray_num(external_dirs_to_restore); i++)
1272
		{
1273
			char	    *external_dir = parray_get(external_dirs_to_restore, i);
1274

1275
			if (strcmp(cell->old_dir, external_dir) == 0)
1276
			{
1277
				/* Swap new dir name with old one, it is used by 2-nd step */
1278
				parray_set(external_dirs_to_restore, i,
1279
						   pgut_strdup(cell->new_dir));
1280
				pfree(external_dir);
1281

1282
				found = true;
1283
				break;
1284
			}
1285
		}
1286
		if (!found)
1287
			elog(ERROR, "--external-mapping option's old directory doesn't "
1288
				 "have an entry in list of external directories of current "
1289
				 "backup: \"%s\"", cell->old_dir);
1290
	}
1291

1292
	/* 2 - all linked directories must be empty */
1293
	for (i = 0; i < parray_num(external_dirs_to_restore); i++)
1294
	{
1295
		char	    *external_dir = (char *) parray_get(external_dirs_to_restore,
1296
														i);
1297

1298
		if (!incremental && !dir_is_empty(external_dir, FIO_DB_HOST))
1299
			elog(ERROR, "External directory is not empty: \"%s\"",
1300
				 external_dir);
1301
	}
1302

1303
	free_dir_list(external_dirs_to_restore);
1304
}
1305

1306
char *
1307
get_external_remap(char *current_dir)
1308
{
1309
	TablespaceListCell *cell;
1310

1311
	for (cell = external_remap_list.head; cell; cell = cell->next)
1312
	{
1313
		char *old_dir = cell->old_dir;
1314

1315
		if (strcmp(old_dir, current_dir) == 0)
1316
			return cell->new_dir;
1317
	}
1318
	return current_dir;
1319
}
1320

1321
/* Parsing states for get_control_value_str() */
1322
#define CONTROL_WAIT_NAME			1
1323
#define CONTROL_INNAME				2
1324
#define CONTROL_WAIT_COLON			3
1325
#define CONTROL_WAIT_VALUE			4
1326
#define CONTROL_INVALUE				5
1327
#define CONTROL_WAIT_NEXT_NAME		6
1328

1329
/*
1330
 * Get value from json-like line "str" of backup_content.control file.
1331
 *
1332
 * The line has the following format:
1333
 *   {"name1":"value1", "name2":"value2"}
1334
 *
1335
 * The value will be returned in "value_int64" as int64.
1336
 *
1337
 * Returns true if the value was found in the line and parsed.
1338
 */
1339
bool
1340
get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory)
1341
{
1342

1343
	char buf_int64[32];
1344

1345
	assert(value_int64);
1346

1347
    /* Set default value */
1348
    *value_int64 = 0;
1349

1350
	if (!get_control_value_str(str, name, buf_int64, sizeof(buf_int64), is_mandatory))
1351
		return false;
1352

1353
	if (!parse_int64(buf_int64, value_int64, 0))
1354
	{
1355
		/* We assume that too big value is -1 */
1356
		if (errno == ERANGE)
1357
			*value_int64 = BYTES_INVALID;
1358
		else
1359
			control_string_bad_format(str);
1360
        return false;
1361
	}
1362

1363
	return true;
1364
}
1365

1366
/*
1367
 * Get value from json-like line "str" of backup_content.control file.
1368
 *
1369
 * The line has the following format:
1370
 *   {"name1":"value1", "name2":"value2"}
1371
 *
1372
 * The value will be returned to "value_str" as string.
1373
 *
1374
 * Returns true if the value was found in the line.
1375
 */
1376

1377
bool
1378
get_control_value_str(const char *str, const char *name,
1379
                      char *value_str, size_t value_str_size, bool is_mandatory)
1380
{
1381
	int			state = CONTROL_WAIT_NAME;
1382
	char	   *name_ptr = (char *) name;
1383
	char	   *buf = (char *) str;
1384
	char 	   *const value_str_start = value_str;
1385

1386
	assert(value_str);
1387
	assert(value_str_size > 0);
1388

1389
	/* Set default value */
1390
	*value_str = '\0';
1391

1392
	while (*buf)
1393
	{
1394
		switch (state)
1395
		{
1396
			case CONTROL_WAIT_NAME:
1397
				if (*buf == '"')
1398
					state = CONTROL_INNAME;
1399
				else if (IsAlpha(*buf))
1400
					control_string_bad_format(str);
1401
				break;
1402
			case CONTROL_INNAME:
1403
				/* Found target field. Parse value. */
1404
				if (*buf == '"')
1405
					state = CONTROL_WAIT_COLON;
1406
				/* Check next field */
1407
				else if (*buf != *name_ptr)
1408
				{
1409
					name_ptr = (char *) name;
1410
					state = CONTROL_WAIT_NEXT_NAME;
1411
				}
1412
				else
1413
					name_ptr++;
1414
				break;
1415
			case CONTROL_WAIT_COLON:
1416
				if (*buf == ':')
1417
					state = CONTROL_WAIT_VALUE;
1418
				else if (!IsSpace(*buf))
1419
					control_string_bad_format(str);
1420
				break;
1421
			case CONTROL_WAIT_VALUE:
1422
				if (*buf == '"')
1423
				{
1424
					state = CONTROL_INVALUE;
1425
				}
1426
				else if (IsAlpha(*buf))
1427
					control_string_bad_format(str);
1428
				break;
1429
			case CONTROL_INVALUE:
1430
				/* Value was parsed, exit */
1431
				if (*buf == '"')
1432
				{
1433
					*value_str = '\0';
1434
					return true;
1435
				}
1436
				else
1437
				{
1438
					/* verify if value_str not exceeds value_str_size limits */
1439
					if (value_str - value_str_start >= value_str_size - 1) {
1440
						elog(ERROR, "Field \"%s\" is out of range in the line %s of the file %s",
1441
							 name, str, DATABASE_FILE_LIST);
1442
					}
1443
					*value_str = *buf;
1444
					value_str++;
1445
				}
1446
				break;
1447
			case CONTROL_WAIT_NEXT_NAME:
1448
				if (*buf == ',')
1449
					state = CONTROL_WAIT_NAME;
1450
				break;
1451
			default:
1452
				/* Should not happen */
1453
				break;
1454
		}
1455

1456
		buf++;
1457
	}
1458

1459
	/* There is no close quotes */
1460
	if (state == CONTROL_INNAME || state == CONTROL_INVALUE)
1461
		control_string_bad_format(str);
1462

1463
	/* Did not find target field */
1464
	if (is_mandatory)
1465
		elog(ERROR, "Field \"%s\" is not found in the line %s of the file %s",
1466
			 name, str, DATABASE_FILE_LIST);
1467
	return false;
1468
}
1469

1470
static void
1471
control_string_bad_format(const char* str)
1472
{
1473
    elog(ERROR, "%s file has invalid format in line %s",
1474
         DATABASE_FILE_LIST, str);
1475
}
1476

1477
/*
1478
 * Check if directory empty.
1479
 */
1480
bool
1481
dir_is_empty(const char *path, fio_location location)
1482
{
1483
	DIR		   *dir;
1484
	struct dirent *dir_ent;
1485

1486
	dir = fio_opendir(path, location);
1487
	if (dir == NULL)
1488
	{
1489
		/* Directory in path doesn't exist */
1490
		if (errno == ENOENT)
1491
			return true;
1492
		elog(ERROR, "Cannot open directory \"%s\": %s", path, strerror(errno));
1493
	}
1494

1495
	errno = 0;
1496
	while ((dir_ent = fio_readdir(dir)))
1497
	{
1498
		/* Skip entries point current dir or parent dir */
1499
		if (strcmp(dir_ent->d_name, ".") == 0 ||
1500
			strcmp(dir_ent->d_name, "..") == 0)
1501
			continue;
1502

1503
		/* Directory is not empty */
1504
		fio_closedir(dir);
1505
		return false;
1506
	}
1507
	if (errno)
1508
		elog(ERROR, "Cannot read directory \"%s\": %s", path, strerror(errno));
1509

1510
	fio_closedir(dir);
1511

1512
	return true;
1513
}
1514

1515
/*
1516
 * Return true if the path is a existing regular file.
1517
 */
1518
bool
1519
fileExists(const char *path, fio_location location)
1520
{
1521
	struct stat buf;
1522

1523
	if (fio_stat(path, &buf, true, location) == -1 && errno == ENOENT)
1524
		return false;
1525
	else if (!S_ISREG(buf.st_mode))
1526
		return false;
1527
	else
1528
		return true;
1529
}
1530

1531
size_t
1532
pgFileSize(const char *path)
1533
{
1534
	struct stat buf;
1535

1536
	if (stat(path, &buf) == -1)
1537
		elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno));
1538

1539
	return buf.st_size;
1540
}
1541

1542
/*
1543
 * Construct parray containing remapped external directories paths
1544
 * from string like /path1:/path2
1545
 */
1546
parray *
1547
make_external_directory_list(const char *colon_separated_dirs, bool remap)
1548
{
1549
	char	   *p;
1550
	parray	   *list = parray_new();
1551
	char	   *tmp = pg_strdup(colon_separated_dirs);
1552

1553
#ifndef WIN32
1554
#define EXTERNAL_DIRECTORY_DELIMITER ":"
1555
#else
1556
#define EXTERNAL_DIRECTORY_DELIMITER ";"
1557
#endif
1558

1559
	p = strtok(tmp, EXTERNAL_DIRECTORY_DELIMITER);
1560
	while(p!=NULL)
1561
	{
1562
		char	   *external_path = pg_strdup(p);
1563

1564
		canonicalize_path(external_path);
1565
		if (is_absolute_path(external_path))
1566
		{
1567
			if (remap)
1568
			{
1569
				char	   *full_path = get_external_remap(external_path);
1570

1571
				if (full_path != external_path)
1572
				{
1573
					full_path = pg_strdup(full_path);
1574
					pfree(external_path);
1575
					external_path = full_path;
1576
				}
1577
			}
1578
			parray_append(list, external_path);
1579
		}
1580
		else
1581
			elog(ERROR, "External directory \"%s\" is not an absolute path",
1582
				 external_path);
1583

1584
		p = strtok(NULL, EXTERNAL_DIRECTORY_DELIMITER);
1585
	}
1586
	pfree(tmp);
1587
	parray_qsort(list, pgCompareString);
1588
	return list;
1589
}
1590

1591
/* Free memory of parray containing strings */
1592
void
1593
free_dir_list(parray *list)
1594
{
1595
	parray_walk(list, pfree);
1596
	parray_free(list);
1597
}
1598

1599
/* Append to string "path_prefix" int "dir_num" */
1600
void
1601
makeExternalDirPathByNum(char *ret_path, const char *path_prefix, const int dir_num)
1602
{
1603
	sprintf(ret_path, "%s%d", path_prefix, dir_num);
1604
}
1605

1606
/* Check if "dir" presents in "dirs_list" */
1607
bool
1608
backup_contains_external(const char *dir, parray *dirs_list)
1609
{
1610
	void *search_result;
1611

1612
	if (!dirs_list) /* There is no external dirs in backup */
1613
		return false;
1614
	search_result = parray_bsearch(dirs_list, dir, pgCompareString);
1615
	return search_result != NULL;
1616
}
1617

1618
/*
1619
 * Print database_map
1620
 */
1621
void
1622
print_database_map(FILE *out, parray *database_map)
1623
{
1624
	int i;
1625

1626
	for (i = 0; i < parray_num(database_map); i++)
1627
	{
1628
		db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, i);
1629

1630
		fio_fprintf(out, "{\"dbOid\":\"%u\", \"datname\":\"%s\"}\n",
1631
				db_entry->dbOid, db_entry->datname);
1632
	}
1633

1634
}
1635

1636
/*
1637
 * Create file 'database_map' and add its meta to backup_files_list
1638
 * NULL check for database_map must be done by the caller.
1639
 */
1640
void
1641
write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_list)
1642
{
1643
	FILE		*fp;
1644
	pgFile		*file;
1645
	char		database_dir[MAXPGPATH];
1646
	char		database_map_path[MAXPGPATH];
1647

1648
	join_path_components(database_dir, backup->root_dir, DATABASE_DIR);
1649
	join_path_components(database_map_path, database_dir, DATABASE_MAP);
1650

1651
	fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST);
1652
	if (fp == NULL)
1653
		elog(ERROR, "Cannot open database map \"%s\": %s", database_map_path,
1654
			 strerror(errno));
1655

1656
	print_database_map(fp, database_map);
1657
	if (fio_fflush(fp) || fio_fclose(fp))
1658
	{
1659
		fio_unlink(database_map_path, FIO_BACKUP_HOST);
1660
		elog(ERROR, "Cannot write database map \"%s\": %s",
1661
			 database_map_path, strerror(errno));
1662
	}
1663

1664
	/* Add metadata to backup_content.control */
1665
	file = pgFileNew(database_map_path, DATABASE_MAP, true, 0,
1666
								 FIO_BACKUP_HOST);
1667
	file->crc = pgFileGetCRC(database_map_path, true, false);
1668
	file->write_size = file->size;
1669
	file->uncompressed_size = file->size;
1670

1671
	parray_append(backup_files_list, file);
1672
}
1673

1674
/*
1675
 * read database map, return NULL if database_map in empty or missing
1676
 */
1677
parray *
1678
read_database_map(pgBackup *backup)
1679
{
1680
	FILE		*fp;
1681
	parray 		*database_map;
1682
	char		buf[MAXPGPATH];
1683
	char		path[MAXPGPATH];
1684
	char		database_map_path[MAXPGPATH];
1685

1686
	join_path_components(path, backup->root_dir, DATABASE_DIR);
1687
	join_path_components(database_map_path, path, DATABASE_MAP);
1688

1689
	fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST);
1690
	if (fp == NULL)
1691
	{
1692
		/* It is NOT ok for database_map to be missing at this point, so
1693
		 * we should error here.
1694
		 * It`s a job of the caller to error if database_map is not empty.
1695
		 */
1696
		elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno));
1697
	}
1698

1699
	database_map = parray_new();
1700

1701
	while (fgets(buf, lengthof(buf), fp))
1702
	{
1703
		char datname[MAXPGPATH];
1704
		int64 dbOid;
1705

1706
		db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry));
1707

1708
        get_control_value_int64(buf, "dbOid", &dbOid, true);
1709
        get_control_value_str(buf, "datname", datname, sizeof(datname), true);
1710

1711
		db_entry->dbOid = dbOid;
1712
		db_entry->datname = pgut_strdup(datname);
1713

1714
		parray_append(database_map, db_entry);
1715
	}
1716

1717
	if (ferror(fp))
1718
			elog(ERROR, "Failed to read from file: \"%s\"", database_map_path);
1719

1720
	fio_close_stream(fp);
1721

1722
	/* Return NULL if file is empty */
1723
	if (parray_num(database_map) == 0)
1724
	{
1725
		parray_free(database_map);
1726
		return NULL;
1727
	}
1728

1729
	return database_map;
1730
}
1731

1732
/*
1733
 * Use it to cleanup tablespaces
1734
 * TODO: Current algorihtm is not very efficient in remote mode,
1735
 * due to round-trip to delete every file.
1736
 */
1737
void
1738
cleanup_tablespace(const char *path)
1739
{
1740
	int i;
1741
	char	fullpath[MAXPGPATH];
1742
	parray *files = parray_new();
1743

1744
	fio_list_dir(files, path, false, false, false, false, false, 0);
1745

1746
	/* delete leaf node first */
1747
	parray_qsort(files, pgFileCompareRelPathWithExternalDesc);
1748

1749
	for (i = 0; i < parray_num(files); i++)
1750
	{
1751
		pgFile	   *file = (pgFile *) parray_get(files, i);
1752

1753
		join_path_components(fullpath, path, file->rel_path);
1754

1755
		fio_delete(file->mode, fullpath, FIO_DB_HOST);
1756
		elog(LOG, "Deleted file \"%s\"", fullpath);
1757
	}
1758

1759
	parray_walk(files, pgFileFree);
1760
	parray_free(files);
1761
}
1762

1763
/*
1764
 * Clear the synchronisation locks in a parray of (pgFile *)'s
1765
 */
1766
void
1767
pfilearray_clear_locks(parray *file_list)
1768
{
1769
	int i;
1770
	for (i = 0; i < parray_num(file_list); i++)
1771
	{
1772
		pgFile *file = (pgFile *) parray_get(file_list, i);
1773
		pg_atomic_clear_flag(&file->lock);
1774
	}
1775
}
1776

1777
static inline bool
1778
is_forkname(char *name, size_t *pos, const char *forkname)
1779
{
1780
	size_t fnlen = strlen(forkname);
1781
	if (strncmp(name + *pos, forkname, fnlen) != 0)
1782
		return false;
1783
	*pos += fnlen;
1784
	return true;
1785
}
1786

1787
#define OIDCHARS 10
1788
#define MAXSEGNO (((uint64_t)1<<32)/RELSEG_SIZE-1)
1789
#define SEGNOCHARS 5 /* when BLCKSZ == (1<<15) */
1790

1791
/* Set forkName if possible */
1792
bool
1793
set_forkname(pgFile *file)
1794
{
1795
	size_t i = 0;
1796
	uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */
1797
	uint64_t segno = 0;
1798

1799
	/* pretend it is not relation file */
1800
	file->relOid = 0;
1801
	file->forkName = none;
1802
	file->is_datafile = false;
1803

1804
	for (i = 0; isdigit(file->name[i]); i++)
1805
	{
1806
		if (i == 0 && file->name[i] == '0')
1807
			return false;
1808
		oid = oid * 10 + file->name[i] - '0';
1809
	}
1810
	if (i == 0 || i > OIDCHARS || oid > UINT32_MAX)
1811
		return false;
1812

1813
	/* usual fork name */
1814
	/* /^\d+_(vm|fsm|init|ptrack)$/ */
1815
	if (is_forkname(file->name, &i, "_vm"))
1816
		file->forkName = vm;
1817
	else if (is_forkname(file->name, &i, "_fsm"))
1818
		file->forkName = fsm;
1819
	else if (is_forkname(file->name, &i, "_init"))
1820
		file->forkName = init;
1821
	else if (is_forkname(file->name, &i, "_ptrack"))
1822
		file->forkName = ptrack;
1823

1824
	/* segment number */
1825
	/* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */
1826
	if (file->name[i] == '.' && isdigit(file->name[i+1]))
1827
	{
1828
		size_t start = i+1;
1829
		for (i++; isdigit(file->name[i]); i++)
1830
		{
1831
			if (i == start && file->name[i] == '0')
1832
				return false;
1833
			segno = segno * 10 + file->name[i] - '0';
1834
		}
1835
		if (i - start > SEGNOCHARS || segno > MAXSEGNO)
1836
			return false;
1837
	}
1838

1839
	/* CFS family fork names */
1840
	if (file->forkName == none &&
1841
		is_forkname(file->name, &i, ".cfm.bck"))
1842
	{
1843
		/* /^\d+(\.\d+)?\.cfm\.bck$/ */
1844
		file->forkName = cfm_bck;
1845
	}
1846
	if (file->forkName == none &&
1847
		is_forkname(file->name, &i, ".bck"))
1848
	{
1849
		/* /^\d+(\.\d+)?\.bck$/ */
1850
		file->forkName = cfs_bck;
1851
	}
1852
	if (file->forkName == none &&
1853
		is_forkname(file->name, &i, ".cfm"))
1854
	{
1855
		/* /^\d+(\.\d+)?.cfm$/ */
1856
		file->forkName = cfm;
1857
	}
1858

1859
	/* If there are excess characters, it is not relation file */
1860
	if (file->name[i] != 0)
1861
	{
1862
		file->forkName = none;
1863
		return false;
1864
	}
1865

1866
	file->relOid = oid;
1867
	file->segno = segno;
1868
	file->is_datafile = file->forkName == none;
1869
	return true;
1870
}
1871