git

1
#define USE_THE_REPOSITORY_VARIABLE
2

3
#include "git-compat-util.h"
4
#include "lockfile.h"
5
#include "bundle.h"
6
#include "environment.h"
7
#include "gettext.h"
8
#include "hex.h"
9
#include "object-store-ll.h"
10
#include "repository.h"
11
#include "object.h"
12
#include "commit.h"
13
#include "diff.h"
14
#include "revision.h"
15
#include "list-objects.h"
16
#include "run-command.h"
17
#include "refs.h"
18
#include "strvec.h"
19
#include "list-objects-filter-options.h"
20
#include "connected.h"
21
#include "write-or-die.h"
22

23
static const char v2_bundle_signature[] = "# v2 git bundle\n";
24
static const char v3_bundle_signature[] = "# v3 git bundle\n";
25
static struct {
26
	int version;
27
	const char *signature;
28
} bundle_sigs[] = {
29
	{ 2, v2_bundle_signature },
30
	{ 3, v3_bundle_signature },
31
};
32

33
void bundle_header_init(struct bundle_header *header)
34
{
35
	struct bundle_header blank = BUNDLE_HEADER_INIT;
36
	memcpy(header, &blank, sizeof(*header));
37
}
38

39
void bundle_header_release(struct bundle_header *header)
40
{
41
	string_list_clear(&header->prerequisites, 1);
42
	string_list_clear(&header->references, 1);
43
	list_objects_filter_release(&header->filter);
44
}
45

46
static int parse_capability(struct bundle_header *header, const char *capability)
47
{
48
	const char *arg;
49
	if (skip_prefix(capability, "object-format=", &arg)) {
50
		int algo = hash_algo_by_name(arg);
51
		if (algo == GIT_HASH_UNKNOWN)
52
			return error(_("unrecognized bundle hash algorithm: %s"), arg);
53
		header->hash_algo = &hash_algos[algo];
54
		return 0;
55
	}
56
	if (skip_prefix(capability, "filter=", &arg)) {
57
		parse_list_objects_filter(&header->filter, arg);
58
		return 0;
59
	}
60
	return error(_("unknown capability '%s'"), capability);
61
}
62

63
static int parse_bundle_signature(struct bundle_header *header, const char *line)
64
{
65
	int i;
66

67
	for (i = 0; i < ARRAY_SIZE(bundle_sigs); i++) {
68
		if (!strcmp(line, bundle_sigs[i].signature)) {
69
			header->version = bundle_sigs[i].version;
70
			return 0;
71
		}
72
	}
73
	return -1;
74
}
75

76
int read_bundle_header_fd(int fd, struct bundle_header *header,
77
			  const char *report_path)
78
{
79
	struct strbuf buf = STRBUF_INIT;
80
	int status = 0;
81

82
	/* The bundle header begins with the signature */
83
	if (strbuf_getwholeline_fd(&buf, fd, '\n') ||
84
	    parse_bundle_signature(header, buf.buf)) {
85
		if (report_path)
86
			error(_("'%s' does not look like a v2 or v3 bundle file"),
87
			      report_path);
88
		status = -1;
89
		goto abort;
90
	}
91

92
	/*
93
	 * The default hash format for bundles is SHA1, unless told otherwise
94
	 * by an "object-format=" capability, which is being handled in
95
	 * `parse_capability()`.
96
	 */
97
	header->hash_algo = &hash_algos[GIT_HASH_SHA1];
98

99
	/* The bundle header ends with an empty line */
100
	while (!strbuf_getwholeline_fd(&buf, fd, '\n') &&
101
	       buf.len && buf.buf[0] != '\n') {
102
		struct object_id oid;
103
		int is_prereq = 0;
104
		const char *p;
105

106
		strbuf_rtrim(&buf);
107

108
		if (header->version == 3 && *buf.buf == '@') {
109
			if (parse_capability(header, buf.buf + 1)) {
110
				status = -1;
111
				break;
112
			}
113
			continue;
114
		}
115

116
		if (*buf.buf == '-') {
117
			is_prereq = 1;
118
			strbuf_remove(&buf, 0, 1);
119
		}
120

121
		/*
122
		 * Tip lines have object name, SP, and refname.
123
		 * Prerequisites have object name that is optionally
124
		 * followed by SP and subject line.
125
		 */
126
		if (parse_oid_hex_algop(buf.buf, &oid, &p, header->hash_algo) ||
127
		    (*p && !isspace(*p)) ||
128
		    (!is_prereq && !*p)) {
129
			if (report_path)
130
				error(_("unrecognized header: %s%s (%d)"),
131
				      (is_prereq ? "-" : ""), buf.buf, (int)buf.len);
132
			status = -1;
133
			break;
134
		} else {
135
			struct object_id *dup = oiddup(&oid);
136
			if (is_prereq)
137
				string_list_append(&header->prerequisites, "")->util = dup;
138
			else
139
				string_list_append(&header->references, p + 1)->util = dup;
140
		}
141
	}
142

143
 abort:
144
	if (status) {
145
		close(fd);
146
		fd = -1;
147
	}
148
	strbuf_release(&buf);
149
	return fd;
150
}
151

152
int read_bundle_header(const char *path, struct bundle_header *header)
153
{
154
	int fd = open(path, O_RDONLY);
155

156
	if (fd < 0)
157
		return error(_("could not open '%s'"), path);
158
	return read_bundle_header_fd(fd, header, path);
159
}
160

161
int is_bundle(const char *path, int quiet)
162
{
163
	struct bundle_header header = BUNDLE_HEADER_INIT;
164
	int fd = open(path, O_RDONLY);
165

166
	if (fd < 0)
167
		return 0;
168
	fd = read_bundle_header_fd(fd, &header, quiet ? NULL : path);
169
	if (fd >= 0)
170
		close(fd);
171
	bundle_header_release(&header);
172
	return (fd >= 0);
173
}
174

175
static int list_refs(struct string_list *r, int argc, const char **argv)
176
{
177
	int i;
178

179
	for (i = 0; i < r->nr; i++) {
180
		struct object_id *oid;
181
		const char *name;
182

183
		if (argc > 1) {
184
			int j;
185
			for (j = 1; j < argc; j++)
186
				if (!strcmp(r->items[i].string, argv[j]))
187
					break;
188
			if (j == argc)
189
				continue;
190
		}
191

192
		oid = r->items[i].util;
193
		name = r->items[i].string;
194
		printf("%s %s\n", oid_to_hex(oid), name);
195
	}
196
	return 0;
197
}
198

199
/* Remember to update object flag allocation in object.h */
200
#define PREREQ_MARK (1u<<16)
201

202
struct string_list_iterator {
203
	struct string_list *list;
204
	size_t cur;
205
};
206

207
static const struct object_id *iterate_ref_map(void *cb_data)
208
{
209
	struct string_list_iterator *iter = cb_data;
210

211
	if (iter->cur >= iter->list->nr)
212
		return NULL;
213

214
	return iter->list->items[iter->cur++].util;
215
}
216

217
int verify_bundle(struct repository *r,
218
		  struct bundle_header *header,
219
		  enum verify_bundle_flags flags)
220
{
221
	/*
222
	 * Do fast check, then if any prereqs are missing then go line by line
223
	 * to be verbose about the errors
224
	 */
225
	struct string_list *p = &header->prerequisites;
226
	int i, ret = 0;
227
	const char *message = _("Repository lacks these prerequisite commits:");
228
	struct string_list_iterator iter = {
229
		.list = p,
230
	};
231
	struct check_connected_options opts = {
232
		.quiet = 1,
233
	};
234

235
	if (!r || !r->objects || !r->objects->odb)
236
		return error(_("need a repository to verify a bundle"));
237

238
	for (i = 0; i < p->nr; i++) {
239
		struct string_list_item *e = p->items + i;
240
		const char *name = e->string;
241
		struct object_id *oid = e->util;
242
		struct object *o = parse_object(r, oid);
243
		if (o)
244
			continue;
245
		ret++;
246
		if (flags & VERIFY_BUNDLE_QUIET)
247
			continue;
248
		if (ret == 1)
249
			error("%s", message);
250
		error("%s %s", oid_to_hex(oid), name);
251
	}
252
	if (ret)
253
		goto cleanup;
254

255
	if ((ret = check_connected(iterate_ref_map, &iter, &opts)))
256
		error(_("some prerequisite commits exist in the object store, "
257
			"but are not connected to the repository's history"));
258

259
	/* TODO: preserve this verbose language. */
260
	if (flags & VERIFY_BUNDLE_VERBOSE) {
261
		struct string_list *r;
262

263
		r = &header->references;
264
		printf_ln(Q_("The bundle contains this ref:",
265
			     "The bundle contains these %"PRIuMAX" refs:",
266
			     r->nr),
267
			  (uintmax_t)r->nr);
268
		list_refs(r, 0, NULL);
269

270
		r = &header->prerequisites;
271
		if (!r->nr) {
272
			printf_ln(_("The bundle records a complete history."));
273
		} else {
274
			printf_ln(Q_("The bundle requires this ref:",
275
				     "The bundle requires these %"PRIuMAX" refs:",
276
				     r->nr),
277
				  (uintmax_t)r->nr);
278
			list_refs(r, 0, NULL);
279
		}
280

281
		printf_ln(_("The bundle uses this hash algorithm: %s"),
282
			  header->hash_algo->name);
283
		if (header->filter.choice)
284
			printf_ln(_("The bundle uses this filter: %s"),
285
				  list_objects_filter_spec(&header->filter));
286
	}
287
cleanup:
288
	return ret;
289
}
290

291
int list_bundle_refs(struct bundle_header *header, int argc, const char **argv)
292
{
293
	return list_refs(&header->references, argc, argv);
294
}
295

296
static int is_tag_in_date_range(struct object *tag, struct rev_info *revs)
297
{
298
	unsigned long size;
299
	enum object_type type;
300
	char *buf = NULL, *line, *lineend;
301
	timestamp_t date;
302
	int result = 1;
303

304
	if (revs->max_age == -1 && revs->min_age == -1)
305
		goto out;
306

307
	buf = repo_read_object_file(the_repository, &tag->oid, &type, &size);
308
	if (!buf)
309
		goto out;
310
	line = memmem(buf, size, "\ntagger ", 8);
311
	if (!line++)
312
		goto out;
313
	lineend = memchr(line, '\n', buf + size - line);
314
	line = memchr(line, '>', lineend ? lineend - line : buf + size - line);
315
	if (!line++)
316
		goto out;
317
	date = parse_timestamp(line, NULL, 10);
318
	result = (revs->max_age == -1 || revs->max_age < date) &&
319
		(revs->min_age == -1 || revs->min_age > date);
320
out:
321
	free(buf);
322
	return result;
323
}
324

325

326
/* Write the pack data to bundle_fd */
327
static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *pack_options)
328
{
329
	struct child_process pack_objects = CHILD_PROCESS_INIT;
330
	int i;
331

332
	strvec_pushl(&pack_objects.args,
333
		     "pack-objects",
334
		     "--stdout", "--thin", "--delta-base-offset",
335
		     NULL);
336
	strvec_pushv(&pack_objects.args, pack_options->v);
337
	if (revs->filter.choice)
338
		strvec_pushf(&pack_objects.args, "--filter=%s",
339
			     list_objects_filter_spec(&revs->filter));
340
	pack_objects.in = -1;
341
	pack_objects.out = bundle_fd;
342
	pack_objects.git_cmd = 1;
343

344
	/*
345
	 * start_command() will close our descriptor if it's >1. Duplicate it
346
	 * to avoid surprising the caller.
347
	 */
348
	if (pack_objects.out > 1) {
349
		pack_objects.out = dup(pack_objects.out);
350
		if (pack_objects.out < 0) {
351
			error_errno(_("unable to dup bundle descriptor"));
352
			child_process_clear(&pack_objects);
353
			return -1;
354
		}
355
	}
356

357
	if (start_command(&pack_objects))
358
		return error(_("Could not spawn pack-objects"));
359

360
	for (i = 0; i < revs->pending.nr; i++) {
361
		struct object *object = revs->pending.objects[i].item;
362
		if (object->flags & UNINTERESTING)
363
			write_or_die(pack_objects.in, "^", 1);
364
		write_or_die(pack_objects.in, oid_to_hex(&object->oid), the_hash_algo->hexsz);
365
		write_or_die(pack_objects.in, "\n", 1);
366
	}
367
	close(pack_objects.in);
368
	if (finish_command(&pack_objects))
369
		return error(_("pack-objects died"));
370
	return 0;
371
}
372

373
/*
374
 * Write out bundle refs based on the tips already
375
 * parsed into revs.pending. As a side effect, may
376
 * manipulate revs.pending to include additional
377
 * necessary objects (like tags).
378
 *
379
 * Returns the number of refs written, or negative
380
 * on error.
381
 */
382
static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
383
{
384
	int i;
385
	int ref_count = 0;
386

387
	for (i = 0; i < revs->pending.nr; i++) {
388
		struct object_array_entry *e = revs->pending.objects + i;
389
		struct object_id oid;
390
		char *ref;
391
		const char *display_ref;
392
		int flag;
393

394
		if (e->item->flags & UNINTERESTING)
395
			continue;
396
		if (repo_dwim_ref(the_repository, e->name, strlen(e->name),
397
				  &oid, &ref, 0) != 1)
398
			goto skip_write_ref;
399
		if (refs_read_ref_full(get_main_ref_store(the_repository), e->name, RESOLVE_REF_READING, &oid, &flag))
400
			flag = 0;
401
		display_ref = (flag & REF_ISSYMREF) ? e->name : ref;
402

403
		if (e->item->type == OBJ_TAG &&
404
				!is_tag_in_date_range(e->item, revs)) {
405
			e->item->flags |= UNINTERESTING;
406
			goto skip_write_ref;
407
		}
408

409
		/*
410
		 * Make sure the refs we wrote out is correct; --max-count and
411
		 * other limiting options could have prevented all the tips
412
		 * from getting output.
413
		 *
414
		 * Non commit objects such as tags and blobs do not have
415
		 * this issue as they are not affected by those extra
416
		 * constraints.
417
		 */
418
		if (!(e->item->flags & SHOWN) && e->item->type == OBJ_COMMIT) {
419
			warning(_("ref '%s' is excluded by the rev-list options"),
420
				e->name);
421
			goto skip_write_ref;
422
		}
423
		/*
424
		 * If you run "git bundle create bndl v1.0..v2.0", the
425
		 * name of the positive ref is "v2.0" but that is the
426
		 * commit that is referenced by the tag, and not the tag
427
		 * itself.
428
		 */
429
		if (!oideq(&oid, &e->item->oid)) {
430
			/*
431
			 * Is this the positive end of a range expressed
432
			 * in terms of a tag (e.g. v2.0 from the range
433
			 * "v1.0..v2.0")?
434
			 */
435
			struct commit *one = lookup_commit_reference(revs->repo, &oid);
436
			struct object *obj;
437

438
			if (e->item == &(one->object)) {
439
				/*
440
				 * Need to include e->name as an
441
				 * independent ref to the pack-objects
442
				 * input, so that the tag is included
443
				 * in the output; otherwise we would
444
				 * end up triggering "empty bundle"
445
				 * error.
446
				 */
447
				obj = parse_object_or_die(&oid, e->name);
448
				obj->flags |= SHOWN;
449
				add_pending_object(revs, obj, e->name);
450
			}
451
			goto skip_write_ref;
452
		}
453

454
		ref_count++;
455
		write_or_die(bundle_fd, oid_to_hex(&e->item->oid), the_hash_algo->hexsz);
456
		write_or_die(bundle_fd, " ", 1);
457
		write_or_die(bundle_fd, display_ref, strlen(display_ref));
458
		write_or_die(bundle_fd, "\n", 1);
459
 skip_write_ref:
460
		free(ref);
461
	}
462

463
	/* end header */
464
	write_or_die(bundle_fd, "\n", 1);
465
	return ref_count;
466
}
467

468
struct bundle_prerequisites_info {
469
	struct object_array *pending;
470
	int fd;
471
};
472

473
static void write_bundle_prerequisites(struct commit *commit, void *data)
474
{
475
	struct bundle_prerequisites_info *bpi = data;
476
	struct object *object;
477
	struct pretty_print_context ctx = { 0 };
478
	struct strbuf buf = STRBUF_INIT;
479

480
	if (!(commit->object.flags & BOUNDARY))
481
		return;
482
	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
483
	write_or_die(bpi->fd, buf.buf, buf.len);
484

485
	ctx.fmt = CMIT_FMT_ONELINE;
486
	ctx.output_encoding = get_log_output_encoding();
487
	strbuf_reset(&buf);
488
	pretty_print_commit(&ctx, commit, &buf);
489
	strbuf_trim(&buf);
490

491
	object = (struct object *)commit;
492
	object->flags |= UNINTERESTING;
493
	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
494
				   NULL);
495
	strbuf_addch(&buf, '\n');
496
	write_or_die(bpi->fd, buf.buf, buf.len);
497
	strbuf_release(&buf);
498
}
499

500
int create_bundle(struct repository *r, const char *path,
501
		  int argc, const char **argv, struct strvec *pack_options, int version)
502
{
503
	struct lock_file lock = LOCK_INIT;
504
	int bundle_fd = -1;
505
	int bundle_to_stdout;
506
	int ref_count = 0;
507
	struct rev_info revs, revs_copy;
508
	int min_version = 2;
509
	struct bundle_prerequisites_info bpi;
510
	int ret;
511
	int i;
512

513
	/* init revs to list objects for pack-objects later */
514
	save_commit_buffer = 0;
515
	repo_init_revisions(r, &revs, NULL);
516

517
	/*
518
	 * Pre-initialize the '--objects' flag so we can parse a
519
	 * --filter option successfully.
520
	 */
521
	revs.tree_objects = revs.blob_objects = 1;
522

523
	argc = setup_revisions(argc, argv, &revs, NULL);
524

525
	/*
526
	 * Reasons to require version 3:
527
	 *
528
	 * 1. @object-format is required because our hash algorithm is not
529
	 *    SHA1.
530
	 * 2. @filter is required because we parsed an object filter.
531
	 */
532
	if (the_hash_algo != &hash_algos[GIT_HASH_SHA1] || revs.filter.choice)
533
		min_version = 3;
534

535
	if (argc > 1) {
536
		ret = error(_("unrecognized argument: %s"), argv[1]);
537
		goto out;
538
	}
539

540
	bundle_to_stdout = !strcmp(path, "-");
541
	if (bundle_to_stdout)
542
		bundle_fd = 1;
543
	else
544
		bundle_fd = hold_lock_file_for_update(&lock, path,
545
						      LOCK_DIE_ON_ERROR);
546

547
	if (version == -1)
548
		version = min_version;
549

550
	if (version < 2 || version > 3) {
551
		die(_("unsupported bundle version %d"), version);
552
	} else if (version < min_version) {
553
		die(_("cannot write bundle version %d with algorithm %s"), version, the_hash_algo->name);
554
	} else if (version == 2) {
555
		write_or_die(bundle_fd, v2_bundle_signature, strlen(v2_bundle_signature));
556
	} else {
557
		const char *capability = "@object-format=";
558
		write_or_die(bundle_fd, v3_bundle_signature, strlen(v3_bundle_signature));
559
		write_or_die(bundle_fd, capability, strlen(capability));
560
		write_or_die(bundle_fd, the_hash_algo->name, strlen(the_hash_algo->name));
561
		write_or_die(bundle_fd, "\n", 1);
562

563
		if (revs.filter.choice) {
564
			const char *value = expand_list_objects_filter_spec(&revs.filter);
565
			capability = "@filter=";
566
			write_or_die(bundle_fd, capability, strlen(capability));
567
			write_or_die(bundle_fd, value, strlen(value));
568
			write_or_die(bundle_fd, "\n", 1);
569
		}
570
	}
571

572
	/* save revs.pending in revs_copy for later use */
573
	memcpy(&revs_copy, &revs, sizeof(revs));
574
	revs_copy.pending.nr = 0;
575
	revs_copy.pending.alloc = 0;
576
	revs_copy.pending.objects = NULL;
577
	for (i = 0; i < revs.pending.nr; i++) {
578
		struct object_array_entry *e = revs.pending.objects + i;
579
		if (e)
580
			add_object_array_with_path(e->item, e->name,
581
						   &revs_copy.pending,
582
						   e->mode, e->path);
583
	}
584

585
	/* write prerequisites */
586
	revs.boundary = 1;
587
	if (prepare_revision_walk(&revs))
588
		die("revision walk setup failed");
589
	bpi.fd = bundle_fd;
590
	bpi.pending = &revs_copy.pending;
591

592
	/*
593
	 * Remove any object walking here. We only care about commits and
594
	 * tags here. The revs_copy has the right instances of these values.
595
	 */
596
	revs.blob_objects = revs.tree_objects = 0;
597
	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
598
	object_array_remove_duplicates(&revs_copy.pending);
599

600
	/* write bundle refs */
601
	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
602
	if (!ref_count) {
603
		die(_("Refusing to create empty bundle."));
604
	} else if (ref_count < 0) {
605
		ret = -1;
606
		goto out;
607
	}
608

609
	/* write pack */
610
	if (write_pack_data(bundle_fd, &revs_copy, pack_options)) {
611
		ret = -1;
612
		goto out;
613
	}
614

615
	if (!bundle_to_stdout) {
616
		if (commit_lock_file(&lock))
617
			die_errno(_("cannot create '%s'"), path);
618
	}
619

620
	ret = 0;
621

622
out:
623
	object_array_clear(&revs_copy.pending);
624
	release_revisions(&revs);
625
	rollback_lock_file(&lock);
626
	return ret;
627
}
628

629
int unbundle(struct repository *r, struct bundle_header *header,
630
	     int bundle_fd, struct strvec *extra_index_pack_args,
631
	     enum verify_bundle_flags flags)
632
{
633
	struct child_process ip = CHILD_PROCESS_INIT;
634

635
	if (verify_bundle(r, header, flags))
636
		return -1;
637

638
	strvec_pushl(&ip.args, "index-pack", "--fix-thin", "--stdin", NULL);
639

640
	/* If there is a filter, then we need to create the promisor pack. */
641
	if (header->filter.choice)
642
		strvec_push(&ip.args, "--promisor=from-bundle");
643

644
	if (flags & VERIFY_BUNDLE_FSCK)
645
		strvec_push(&ip.args, "--fsck-objects");
646

647
	if (extra_index_pack_args) {
648
		strvec_pushv(&ip.args, extra_index_pack_args->v);
649
		strvec_clear(extra_index_pack_args);
650
	}
651

652
	ip.in = bundle_fd;
653
	ip.no_stdout = 1;
654
	ip.git_cmd = 1;
655
	if (run_command(&ip))
656
		return error(_("index-pack died"));
657
	return 0;
658
}
659