git

1
#define USE_THE_REPOSITORY_VARIABLE
2

3
#include "git-compat-util.h"
4
#include "repository.h"
5
#include "config.h"
6
#include "date.h"
7
#include "environment.h"
8
#include "gettext.h"
9
#include "hex.h"
10
#include "lockfile.h"
11
#include "refs.h"
12
#include "pkt-line.h"
13
#include "commit.h"
14
#include "tag.h"
15
#include "pack.h"
16
#include "sideband.h"
17
#include "fetch-pack.h"
18
#include "remote.h"
19
#include "run-command.h"
20
#include "connect.h"
21
#include "trace2.h"
22
#include "version.h"
23
#include "oid-array.h"
24
#include "oidset.h"
25
#include "packfile.h"
26
#include "object-store-ll.h"
27
#include "path.h"
28
#include "connected.h"
29
#include "fetch-negotiator.h"
30
#include "fsck.h"
31
#include "shallow.h"
32
#include "commit-reach.h"
33
#include "commit-graph.h"
34
#include "sigchain.h"
35
#include "mergesort.h"
36

37
static int transfer_unpack_limit = -1;
38
static int fetch_unpack_limit = -1;
39
static int unpack_limit = 100;
40
static int prefer_ofs_delta = 1;
41
static int no_done;
42
static int deepen_since_ok;
43
static int deepen_not_ok;
44
static int fetch_fsck_objects = -1;
45
static int transfer_fsck_objects = -1;
46
static int agent_supported;
47
static int server_supports_filtering;
48
static int advertise_sid;
49
static struct shallow_lock shallow_lock;
50
static const char *alternate_shallow_file;
51
static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
52
static struct strbuf fsck_msg_types = STRBUF_INIT;
53
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
54

55
/* Remember to update object flag allocation in object.h */
56
#define COMPLETE	(1U << 0)
57
#define ALTERNATE	(1U << 1)
58
#define COMMON		(1U << 6)
59
#define REACH_SCRATCH	(1U << 7)
60

61
/*
62
 * After sending this many "have"s if we do not get any new ACK , we
63
 * give up traversing our history.
64
 */
65
#define MAX_IN_VAIN 256
66

67
static int multi_ack, use_sideband;
68
/* Allow specifying sha1 if it is a ref tip. */
69
#define ALLOW_TIP_SHA1	01
70
/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
71
#define ALLOW_REACHABLE_SHA1	02
72
static unsigned int allow_unadvertised_object_request;
73

74
__attribute__((format (printf, 2, 3)))
75
static inline void print_verbose(const struct fetch_pack_args *args,
76
				 const char *fmt, ...)
77
{
78
	va_list params;
79

80
	if (!args->verbose)
81
		return;
82

83
	va_start(params, fmt);
84
	vfprintf(stderr, fmt, params);
85
	va_end(params);
86
	fputc('\n', stderr);
87
}
88

89
struct alternate_object_cache {
90
	struct object **items;
91
	size_t nr, alloc;
92
};
93

94
static void cache_one_alternate(const struct object_id *oid,
95
				void *vcache)
96
{
97
	struct alternate_object_cache *cache = vcache;
98
	struct object *obj = parse_object(the_repository, oid);
99

100
	if (!obj || (obj->flags & ALTERNATE))
101
		return;
102

103
	obj->flags |= ALTERNATE;
104
	ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
105
	cache->items[cache->nr++] = obj;
106
}
107

108
static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
109
				      void (*cb)(struct fetch_negotiator *,
110
						 struct object *))
111
{
112
	static int initialized;
113
	static struct alternate_object_cache cache;
114
	size_t i;
115

116
	if (!initialized) {
117
		for_each_alternate_ref(cache_one_alternate, &cache);
118
		initialized = 1;
119
	}
120

121
	for (i = 0; i < cache.nr; i++)
122
		cb(negotiator, cache.items[i]);
123
}
124

125
static struct commit *deref_without_lazy_fetch_extended(const struct object_id *oid,
126
							int mark_tags_complete,
127
							enum object_type *type,
128
							unsigned int oi_flags)
129
{
130
	struct object_info info = { .typep = type };
131
	struct commit *commit;
132

133
	commit = lookup_commit_in_graph(the_repository, oid);
134
	if (commit)
135
		return commit;
136

137
	while (1) {
138
		if (oid_object_info_extended(the_repository, oid, &info,
139
					     oi_flags))
140
			return NULL;
141
		if (*type == OBJ_TAG) {
142
			struct tag *tag = (struct tag *)
143
				parse_object(the_repository, oid);
144

145
			if (!tag->tagged)
146
				return NULL;
147
			if (mark_tags_complete)
148
				tag->object.flags |= COMPLETE;
149
			oid = &tag->tagged->oid;
150
		} else {
151
			break;
152
		}
153
	}
154

155
	if (*type == OBJ_COMMIT) {
156
		struct commit *commit = lookup_commit(the_repository, oid);
157
		if (!commit || repo_parse_commit(the_repository, commit))
158
			return NULL;
159
		return commit;
160
	}
161

162
	return NULL;
163
}
164

165

166
static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
167
					       int mark_tags_complete)
168
{
169
	enum object_type type;
170
	unsigned flags = OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK;
171
	return deref_without_lazy_fetch_extended(oid, mark_tags_complete,
172
						 &type, flags);
173
}
174

175
static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
176
			       const struct object_id *oid)
177
{
178
	struct commit *c = deref_without_lazy_fetch(oid, 0);
179

180
	if (c)
181
		negotiator->add_tip(negotiator, c);
182
	return 0;
183
}
184

185
static int rev_list_insert_ref_oid(const char *refname UNUSED,
186
				   const char *referent UNUSED,
187
				   const struct object_id *oid,
188
				   int flag UNUSED,
189
				   void *cb_data)
190
{
191
	return rev_list_insert_ref(cb_data, oid);
192
}
193

194
enum ack_type {
195
	NAK = 0,
196
	ACK,
197
	ACK_continue,
198
	ACK_common,
199
	ACK_ready
200
};
201

202
static void consume_shallow_list(struct fetch_pack_args *args,
203
				 struct packet_reader *reader)
204
{
205
	if (args->stateless_rpc && args->deepen) {
206
		/* If we sent a depth we will get back "duplicate"
207
		 * shallow and unshallow commands every time there
208
		 * is a block of have lines exchanged.
209
		 */
210
		while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
211
			if (starts_with(reader->line, "shallow "))
212
				continue;
213
			if (starts_with(reader->line, "unshallow "))
214
				continue;
215
			die(_("git fetch-pack: expected shallow list"));
216
		}
217
		if (reader->status != PACKET_READ_FLUSH)
218
			die(_("git fetch-pack: expected a flush packet after shallow list"));
219
	}
220
}
221

222
static enum ack_type get_ack(struct packet_reader *reader,
223
			     struct object_id *result_oid)
224
{
225
	int len;
226
	const char *arg;
227

228
	if (packet_reader_read(reader) != PACKET_READ_NORMAL)
229
		die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
230
	len = reader->pktlen;
231

232
	if (!strcmp(reader->line, "NAK"))
233
		return NAK;
234
	if (skip_prefix(reader->line, "ACK ", &arg)) {
235
		const char *p;
236
		if (!parse_oid_hex(arg, result_oid, &p)) {
237
			len -= p - reader->line;
238
			if (len < 1)
239
				return ACK;
240
			if (strstr(p, "continue"))
241
				return ACK_continue;
242
			if (strstr(p, "common"))
243
				return ACK_common;
244
			if (strstr(p, "ready"))
245
				return ACK_ready;
246
			return ACK;
247
		}
248
	}
249
	die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
250
}
251

252
static void send_request(struct fetch_pack_args *args,
253
			 int fd, struct strbuf *buf)
254
{
255
	if (args->stateless_rpc) {
256
		send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
257
		packet_flush(fd);
258
	} else {
259
		if (write_in_full(fd, buf->buf, buf->len) < 0)
260
			die_errno(_("unable to write to remote"));
261
	}
262
}
263

264
static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
265
					struct object *obj)
266
{
267
	rev_list_insert_ref(negotiator, &obj->oid);
268
}
269

270
#define INITIAL_FLUSH 16
271
#define PIPESAFE_FLUSH 32
272
#define LARGE_FLUSH 16384
273

274
static int next_flush(int stateless_rpc, int count)
275
{
276
	if (stateless_rpc) {
277
		if (count < LARGE_FLUSH)
278
			count <<= 1;
279
		else
280
			count = count * 11 / 10;
281
	} else {
282
		if (count < PIPESAFE_FLUSH)
283
			count <<= 1;
284
		else
285
			count += PIPESAFE_FLUSH;
286
	}
287
	return count;
288
}
289

290
static void mark_tips(struct fetch_negotiator *negotiator,
291
		      const struct oid_array *negotiation_tips)
292
{
293
	int i;
294

295
	if (!negotiation_tips) {
296
		refs_for_each_rawref(get_main_ref_store(the_repository),
297
				     rev_list_insert_ref_oid, negotiator);
298
		return;
299
	}
300

301
	for (i = 0; i < negotiation_tips->nr; i++)
302
		rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
303
	return;
304
}
305

306
static void send_filter(struct fetch_pack_args *args,
307
			struct strbuf *req_buf,
308
			int server_supports_filter)
309
{
310
	if (args->filter_options.choice) {
311
		const char *spec =
312
			expand_list_objects_filter_spec(&args->filter_options);
313
		if (server_supports_filter) {
314
			print_verbose(args, _("Server supports filter"));
315
			packet_buf_write(req_buf, "filter %s", spec);
316
			trace2_data_string("fetch", the_repository,
317
					   "filter/effective", spec);
318
		} else {
319
			warning("filtering not recognized by server, ignoring");
320
			trace2_data_string("fetch", the_repository,
321
					   "filter/unsupported", spec);
322
		}
323
	} else {
324
		trace2_data_string("fetch", the_repository,
325
				   "filter/none", "");
326
	}
327
}
328

329
static int find_common(struct fetch_negotiator *negotiator,
330
		       struct fetch_pack_args *args,
331
		       int fd[2], struct object_id *result_oid,
332
		       struct ref *refs)
333
{
334
	int fetching;
335
	int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
336
	int negotiation_round = 0, haves = 0;
337
	const struct object_id *oid;
338
	unsigned in_vain = 0;
339
	int got_continue = 0;
340
	int got_ready = 0;
341
	struct strbuf req_buf = STRBUF_INIT;
342
	size_t state_len = 0;
343
	struct packet_reader reader;
344

345
	if (args->stateless_rpc && multi_ack == 1)
346
		die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");
347

348
	packet_reader_init(&reader, fd[0], NULL, 0,
349
			   PACKET_READ_CHOMP_NEWLINE |
350
			   PACKET_READ_DIE_ON_ERR_PACKET);
351

352
	mark_tips(negotiator, args->negotiation_tips);
353
	for_each_cached_alternate(negotiator, insert_one_alternate_object);
354

355
	fetching = 0;
356
	for ( ; refs ; refs = refs->next) {
357
		struct object_id *remote = &refs->old_oid;
358
		const char *remote_hex;
359
		struct object *o;
360

361
		if (!args->refetch) {
362
			/*
363
			* If that object is complete (i.e. it is an ancestor of a
364
			* local ref), we tell them we have it but do not have to
365
			* tell them about its ancestors, which they already know
366
			* about.
367
			*
368
			* We use lookup_object here because we are only
369
			* interested in the case we *know* the object is
370
			* reachable and we have already scanned it.
371
			*/
372
			if (((o = lookup_object(the_repository, remote)) != NULL) &&
373
					(o->flags & COMPLETE)) {
374
				continue;
375
			}
376
		}
377

378
		remote_hex = oid_to_hex(remote);
379
		if (!fetching) {
380
			struct strbuf c = STRBUF_INIT;
381
			if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
382
			if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
383
			if (no_done)            strbuf_addstr(&c, " no-done");
384
			if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
385
			if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
386
			if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
387
			if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
388
			if (args->no_progress)   strbuf_addstr(&c, " no-progress");
389
			if (args->include_tag)   strbuf_addstr(&c, " include-tag");
390
			if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
391
			if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
392
			if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
393
			if (agent_supported)    strbuf_addf(&c, " agent=%s",
394
							    git_user_agent_sanitized());
395
			if (advertise_sid)
396
				strbuf_addf(&c, " session-id=%s", trace2_session_id());
397
			if (args->filter_options.choice)
398
				strbuf_addstr(&c, " filter");
399
			packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
400
			strbuf_release(&c);
401
		} else
402
			packet_buf_write(&req_buf, "want %s\n", remote_hex);
403
		fetching++;
404
	}
405

406
	if (!fetching) {
407
		strbuf_release(&req_buf);
408
		packet_flush(fd[1]);
409
		return 1;
410
	}
411

412
	if (is_repository_shallow(the_repository))
413
		write_shallow_commits(&req_buf, 1, NULL);
414
	if (args->depth > 0)
415
		packet_buf_write(&req_buf, "deepen %d", args->depth);
416
	if (args->deepen_since) {
417
		timestamp_t max_age = approxidate(args->deepen_since);
418
		packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
419
	}
420
	if (args->deepen_not) {
421
		int i;
422
		for (i = 0; i < args->deepen_not->nr; i++) {
423
			struct string_list_item *s = args->deepen_not->items + i;
424
			packet_buf_write(&req_buf, "deepen-not %s", s->string);
425
		}
426
	}
427
	send_filter(args, &req_buf, server_supports_filtering);
428
	packet_buf_flush(&req_buf);
429
	state_len = req_buf.len;
430

431
	if (args->deepen) {
432
		const char *arg;
433
		struct object_id oid;
434

435
		send_request(args, fd[1], &req_buf);
436
		while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
437
			if (skip_prefix(reader.line, "shallow ", &arg)) {
438
				if (get_oid_hex(arg, &oid))
439
					die(_("invalid shallow line: %s"), reader.line);
440
				register_shallow(the_repository, &oid);
441
				continue;
442
			}
443
			if (skip_prefix(reader.line, "unshallow ", &arg)) {
444
				if (get_oid_hex(arg, &oid))
445
					die(_("invalid unshallow line: %s"), reader.line);
446
				if (!lookup_object(the_repository, &oid))
447
					die(_("object not found: %s"), reader.line);
448
				/* make sure that it is parsed as shallow */
449
				if (!parse_object(the_repository, &oid))
450
					die(_("error in object: %s"), reader.line);
451
				if (unregister_shallow(&oid))
452
					die(_("no shallow found: %s"), reader.line);
453
				continue;
454
			}
455
			die(_("expected shallow/unshallow, got %s"), reader.line);
456
		}
457
	} else if (!args->stateless_rpc)
458
		send_request(args, fd[1], &req_buf);
459

460
	if (!args->stateless_rpc) {
461
		/* If we aren't using the stateless-rpc interface
462
		 * we don't need to retain the headers.
463
		 */
464
		strbuf_setlen(&req_buf, 0);
465
		state_len = 0;
466
	}
467

468
	trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
469
	flushes = 0;
470
	retval = -1;
471
	while ((oid = negotiator->next(negotiator))) {
472
		packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
473
		print_verbose(args, "have %s", oid_to_hex(oid));
474
		in_vain++;
475
		haves++;
476
		if (flush_at <= ++count) {
477
			int ack;
478

479
			negotiation_round++;
480
			trace2_region_enter_printf("negotiation_v0_v1", "round",
481
						   the_repository, "%d",
482
						   negotiation_round);
483
			trace2_data_intmax("negotiation_v0_v1", the_repository,
484
					   "haves_added", haves);
485
			trace2_data_intmax("negotiation_v0_v1", the_repository,
486
					   "in_vain", in_vain);
487
			haves = 0;
488
			packet_buf_flush(&req_buf);
489
			send_request(args, fd[1], &req_buf);
490
			strbuf_setlen(&req_buf, state_len);
491
			flushes++;
492
			flush_at = next_flush(args->stateless_rpc, count);
493

494
			/*
495
			 * We keep one window "ahead" of the other side, and
496
			 * will wait for an ACK only on the next one
497
			 */
498
			if (!args->stateless_rpc && count == INITIAL_FLUSH)
499
				continue;
500

501
			consume_shallow_list(args, &reader);
502
			do {
503
				ack = get_ack(&reader, result_oid);
504
				if (ack)
505
					print_verbose(args, _("got %s %d %s"), "ack",
506
						      ack, oid_to_hex(result_oid));
507
				switch (ack) {
508
				case ACK:
509
					trace2_region_leave_printf("negotiation_v0_v1", "round",
510
								   the_repository, "%d",
511
								   negotiation_round);
512
					flushes = 0;
513
					multi_ack = 0;
514
					retval = 0;
515
					goto done;
516
				case ACK_common:
517
				case ACK_ready:
518
				case ACK_continue: {
519
					struct commit *commit =
520
						lookup_commit(the_repository,
521
							      result_oid);
522
					int was_common;
523

524
					if (!commit)
525
						die(_("invalid commit %s"), oid_to_hex(result_oid));
526
					was_common = negotiator->ack(negotiator, commit);
527
					if (args->stateless_rpc
528
					 && ack == ACK_common
529
					 && !was_common) {
530
						/* We need to replay the have for this object
531
						 * on the next RPC request so the peer knows
532
						 * it is in common with us.
533
						 */
534
						const char *hex = oid_to_hex(result_oid);
535
						packet_buf_write(&req_buf, "have %s\n", hex);
536
						state_len = req_buf.len;
537
						haves++;
538
						/*
539
						 * Reset in_vain because an ack
540
						 * for this commit has not been
541
						 * seen.
542
						 */
543
						in_vain = 0;
544
					} else if (!args->stateless_rpc
545
						   || ack != ACK_common)
546
						in_vain = 0;
547
					retval = 0;
548
					got_continue = 1;
549
					if (ack == ACK_ready)
550
						got_ready = 1;
551
					break;
552
					}
553
				}
554
			} while (ack);
555
			flushes--;
556
			trace2_region_leave_printf("negotiation_v0_v1", "round",
557
						   the_repository, "%d",
558
						   negotiation_round);
559
			if (got_continue && MAX_IN_VAIN < in_vain) {
560
				print_verbose(args, _("giving up"));
561
				break; /* give up */
562
			}
563
			if (got_ready)
564
				break;
565
		}
566
	}
567
done:
568
	trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
569
	trace2_data_intmax("negotiation_v0_v1", the_repository, "total_rounds",
570
			   negotiation_round);
571
	if (!got_ready || !no_done) {
572
		packet_buf_write(&req_buf, "done\n");
573
		send_request(args, fd[1], &req_buf);
574
	}
575
	print_verbose(args, _("done"));
576
	if (retval != 0) {
577
		multi_ack = 0;
578
		flushes++;
579
	}
580
	strbuf_release(&req_buf);
581

582
	if (!got_ready || !no_done)
583
		consume_shallow_list(args, &reader);
584
	while (flushes || multi_ack) {
585
		int ack = get_ack(&reader, result_oid);
586
		if (ack) {
587
			print_verbose(args, _("got %s (%d) %s"), "ack",
588
				      ack, oid_to_hex(result_oid));
589
			if (ack == ACK)
590
				return 0;
591
			multi_ack = 1;
592
			continue;
593
		}
594
		flushes--;
595
	}
596
	/* it is no error to fetch into a completely empty repo */
597
	return count ? retval : 0;
598
}
599

600
static struct commit_list *complete;
601

602
static int mark_complete(const struct object_id *oid)
603
{
604
	struct commit *commit = deref_without_lazy_fetch(oid, 1);
605

606
	if (commit && !(commit->object.flags & COMPLETE)) {
607
		commit->object.flags |= COMPLETE;
608
		commit_list_insert(commit, &complete);
609
	}
610
	return 0;
611
}
612

613
static int mark_complete_oid(const char *refname UNUSED,
614
			     const char *referent UNUSED,
615
			     const struct object_id *oid,
616
			     int flag UNUSED,
617
			     void *cb_data UNUSED)
618
{
619
	return mark_complete(oid);
620
}
621

622
static void mark_recent_complete_commits(struct fetch_pack_args *args,
623
					 timestamp_t cutoff)
624
{
625
	while (complete && cutoff <= complete->item->date) {
626
		print_verbose(args, _("Marking %s as complete"),
627
			      oid_to_hex(&complete->item->object.oid));
628
		pop_most_recent_commit(&complete, COMPLETE);
629
	}
630
}
631

632
static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
633
{
634
	for (; refs; refs = refs->next)
635
		oidset_insert(oids, &refs->old_oid);
636
}
637

638
static int is_unmatched_ref(const struct ref *ref)
639
{
640
	struct object_id oid;
641
	const char *p;
642
	return	ref->match_status == REF_NOT_MATCHED &&
643
		!parse_oid_hex(ref->name, &oid, &p) &&
644
		*p == '\0' &&
645
		oideq(&oid, &ref->old_oid);
646
}
647

648
static void filter_refs(struct fetch_pack_args *args,
649
			struct ref **refs,
650
			struct ref **sought, int nr_sought)
651
{
652
	struct ref *newlist = NULL;
653
	struct ref **newtail = &newlist;
654
	struct ref *unmatched = NULL;
655
	struct ref *ref, *next;
656
	struct oidset tip_oids = OIDSET_INIT;
657
	int i;
658
	int strict = !(allow_unadvertised_object_request &
659
		       (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
660

661
	i = 0;
662
	for (ref = *refs; ref; ref = next) {
663
		int keep = 0;
664
		next = ref->next;
665

666
		if (starts_with(ref->name, "refs/") &&
667
		    check_refname_format(ref->name, 0)) {
668
			/*
669
			 * trash or a peeled value; do not even add it to
670
			 * unmatched list
671
			 */
672
			free_one_ref(ref);
673
			continue;
674
		} else {
675
			while (i < nr_sought) {
676
				int cmp = strcmp(ref->name, sought[i]->name);
677
				if (cmp < 0)
678
					break; /* definitely do not have it */
679
				else if (cmp == 0) {
680
					keep = 1; /* definitely have it */
681
					sought[i]->match_status = REF_MATCHED;
682
				}
683
				i++;
684
			}
685

686
			if (!keep && args->fetch_all &&
687
			    (!args->deepen || !starts_with(ref->name, "refs/tags/")))
688
				keep = 1;
689
		}
690

691
		if (keep) {
692
			*newtail = ref;
693
			ref->next = NULL;
694
			newtail = &ref->next;
695
		} else {
696
			ref->next = unmatched;
697
			unmatched = ref;
698
		}
699
	}
700

701
	if (strict) {
702
		for (i = 0; i < nr_sought; i++) {
703
			ref = sought[i];
704
			if (!is_unmatched_ref(ref))
705
				continue;
706

707
			add_refs_to_oidset(&tip_oids, unmatched);
708
			add_refs_to_oidset(&tip_oids, newlist);
709
			break;
710
		}
711
	}
712

713
	/* Append unmatched requests to the list */
714
	for (i = 0; i < nr_sought; i++) {
715
		ref = sought[i];
716
		if (!is_unmatched_ref(ref))
717
			continue;
718

719
		if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
720
			ref->match_status = REF_MATCHED;
721
			*newtail = copy_ref(ref);
722
			newtail = &(*newtail)->next;
723
		} else {
724
			ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
725
		}
726
	}
727

728
	oidset_clear(&tip_oids);
729
	free_refs(unmatched);
730

731
	*refs = newlist;
732
}
733

734
static void mark_alternate_complete(struct fetch_negotiator *negotiator UNUSED,
735
				    struct object *obj)
736
{
737
	mark_complete(&obj->oid);
738
}
739

740
/*
741
 * Mark recent commits available locally and reachable from a local ref as
742
 * COMPLETE.
743
 *
744
 * The cutoff time for recency is determined by this heuristic: it is the
745
 * earliest commit time of the objects in refs that are commits and that we know
746
 * the commit time of.
747
 */
748
static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
749
					 struct fetch_pack_args *args,
750
					 struct ref **refs)
751
{
752
	struct ref *ref;
753
	int old_save_commit_buffer = save_commit_buffer;
754
	timestamp_t cutoff = 0;
755

756
	if (args->refetch)
757
		return;
758

759
	save_commit_buffer = 0;
760

761
	trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
762
	for (ref = *refs; ref; ref = ref->next) {
763
		struct commit *commit;
764

765
		commit = lookup_commit_in_graph(the_repository, &ref->old_oid);
766
		if (!commit) {
767
			struct object *o;
768

769
			if (!repo_has_object_file_with_flags(the_repository, &ref->old_oid,
770
							     OBJECT_INFO_QUICK |
771
							     OBJECT_INFO_SKIP_FETCH_OBJECT))
772
				continue;
773
			o = parse_object(the_repository, &ref->old_oid);
774
			if (!o || o->type != OBJ_COMMIT)
775
				continue;
776

777
			commit = (struct commit *)o;
778
		}
779

780
		/*
781
		 * We already have it -- which may mean that we were
782
		 * in sync with the other side at some time after
783
		 * that (it is OK if we guess wrong here).
784
		 */
785
		if (!cutoff || cutoff < commit->date)
786
			cutoff = commit->date;
787
	}
788
	trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
789

790
	/*
791
	 * This block marks all local refs as COMPLETE, and then recursively marks all
792
	 * parents of those refs as COMPLETE.
793
	 */
794
	trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
795
	if (!args->deepen) {
796
		refs_for_each_rawref(get_main_ref_store(the_repository),
797
				     mark_complete_oid, NULL);
798
		for_each_cached_alternate(NULL, mark_alternate_complete);
799
		commit_list_sort_by_date(&complete);
800
		if (cutoff)
801
			mark_recent_complete_commits(args, cutoff);
802
	}
803
	trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);
804

805
	/*
806
	 * Mark all complete remote refs as common refs.
807
	 * Don't mark them common yet; the server has to be told so first.
808
	 */
809
	trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
810
	for (ref = *refs; ref; ref = ref->next) {
811
		struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0);
812

813
		if (!c || !(c->object.flags & COMPLETE))
814
			continue;
815

816
		negotiator->known_common(negotiator, c);
817
	}
818
	trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
819

820
	save_commit_buffer = old_save_commit_buffer;
821
}
822

823
/*
824
 * Returns 1 if every object pointed to by the given remote refs is available
825
 * locally and reachable from a local ref, and 0 otherwise.
826
 */
827
static int everything_local(struct fetch_pack_args *args,
828
			    struct ref **refs)
829
{
830
	struct ref *ref;
831
	int retval;
832

833
	for (retval = 1, ref = *refs; ref ; ref = ref->next) {
834
		const struct object_id *remote = &ref->old_oid;
835
		struct object *o;
836

837
		o = lookup_object(the_repository, remote);
838
		if (!o || !(o->flags & COMPLETE)) {
839
			retval = 0;
840
			print_verbose(args, "want %s (%s)", oid_to_hex(remote),
841
				      ref->name);
842
			continue;
843
		}
844
		print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
845
			      ref->name);
846
	}
847

848
	return retval;
849
}
850

851
static int sideband_demux(int in UNUSED, int out, void *data)
852
{
853
	int *xd = data;
854
	int ret;
855

856
	ret = recv_sideband("fetch-pack", xd[0], out);
857
	close(out);
858
	return ret;
859
}
860

861
static void create_promisor_file(const char *keep_name,
862
				 struct ref **sought, int nr_sought)
863
{
864
	struct strbuf promisor_name = STRBUF_INIT;
865
	int suffix_stripped;
866

867
	strbuf_addstr(&promisor_name, keep_name);
868
	suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
869
	if (!suffix_stripped)
870
		BUG("name of pack lockfile should end with .keep (was '%s')",
871
		    keep_name);
872
	strbuf_addstr(&promisor_name, ".promisor");
873

874
	write_promisor_file(promisor_name.buf, sought, nr_sought);
875

876
	strbuf_release(&promisor_name);
877
}
878

879
static void parse_gitmodules_oids(int fd, struct oidset *gitmodules_oids)
880
{
881
	int len = the_hash_algo->hexsz + 1; /* hash + NL */
882

883
	do {
884
		char hex_hash[GIT_MAX_HEXSZ + 1];
885
		int read_len = read_in_full(fd, hex_hash, len);
886
		struct object_id oid;
887
		const char *end;
888

889
		if (!read_len)
890
			return;
891
		if (read_len != len)
892
			die("invalid length read %d", read_len);
893
		if (parse_oid_hex(hex_hash, &oid, &end) || *end != '\n')
894
			die("invalid hash");
895
		oidset_insert(gitmodules_oids, &oid);
896
	} while (1);
897
}
898

899
static void add_index_pack_keep_option(struct strvec *args)
900
{
901
	char hostname[HOST_NAME_MAX + 1];
902

903
	if (xgethostname(hostname, sizeof(hostname)))
904
		xsnprintf(hostname, sizeof(hostname), "localhost");
905
	strvec_pushf(args, "--keep=fetch-pack %"PRIuMAX " on %s",
906
		     (uintmax_t)getpid(), hostname);
907
}
908

909
/*
910
 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
911
 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
912
 * stored there. (It must be freed by the caller.)
913
 */
914
static int get_pack(struct fetch_pack_args *args,
915
		    int xd[2], struct string_list *pack_lockfiles,
916
		    struct strvec *index_pack_args,
917
		    struct ref **sought, int nr_sought,
918
		    struct oidset *gitmodules_oids)
919
{
920
	struct async demux;
921
	int do_keep = args->keep_pack;
922
	const char *cmd_name;
923
	struct pack_header header;
924
	int pass_header = 0;
925
	struct child_process cmd = CHILD_PROCESS_INIT;
926
	int fsck_objects = 0;
927
	int ret;
928

929
	memset(&demux, 0, sizeof(demux));
930
	if (use_sideband) {
931
		/* xd[] is talking with upload-pack; subprocess reads from
932
		 * xd[0], spits out band#2 to stderr, and feeds us band#1
933
		 * through demux->out.
934
		 */
935
		demux.proc = sideband_demux;
936
		demux.data = xd;
937
		demux.out = -1;
938
		demux.isolate_sigpipe = 1;
939
		if (start_async(&demux))
940
			die(_("fetch-pack: unable to fork off sideband demultiplexer"));
941
	}
942
	else
943
		demux.out = xd[0];
944

945
	if (!args->keep_pack && unpack_limit && !index_pack_args) {
946

947
		if (read_pack_header(demux.out, &header))
948
			die(_("protocol error: bad pack header"));
949
		pass_header = 1;
950
		if (ntohl(header.hdr_entries) < unpack_limit)
951
			do_keep = 0;
952
		else
953
			do_keep = 1;
954
	}
955

956
	if (alternate_shallow_file) {
957
		strvec_push(&cmd.args, "--shallow-file");
958
		strvec_push(&cmd.args, alternate_shallow_file);
959
	}
960

961
	fsck_objects = fetch_pack_fsck_objects();
962

963
	if (do_keep || args->from_promisor || index_pack_args || fsck_objects) {
964
		if (pack_lockfiles || fsck_objects)
965
			cmd.out = -1;
966
		cmd_name = "index-pack";
967
		strvec_push(&cmd.args, cmd_name);
968
		strvec_push(&cmd.args, "--stdin");
969
		if (!args->quiet && !args->no_progress)
970
			strvec_push(&cmd.args, "-v");
971
		if (args->use_thin_pack)
972
			strvec_push(&cmd.args, "--fix-thin");
973
		if ((do_keep || index_pack_args) && (args->lock_pack || unpack_limit))
974
			add_index_pack_keep_option(&cmd.args);
975
		if (!index_pack_args && args->check_self_contained_and_connected)
976
			strvec_push(&cmd.args, "--check-self-contained-and-connected");
977
		else
978
			/*
979
			 * We cannot perform any connectivity checks because
980
			 * not all packs have been downloaded; let the caller
981
			 * have this responsibility.
982
			 */
983
			args->check_self_contained_and_connected = 0;
984

985
		if (args->from_promisor)
986
			/*
987
			 * create_promisor_file() may be called afterwards but
988
			 * we still need index-pack to know that this is a
989
			 * promisor pack. For example, if transfer.fsckobjects
990
			 * is true, index-pack needs to know that .gitmodules
991
			 * is a promisor object (so that it won't complain if
992
			 * it is missing).
993
			 */
994
			strvec_push(&cmd.args, "--promisor");
995
	}
996
	else {
997
		cmd_name = "unpack-objects";
998
		strvec_push(&cmd.args, cmd_name);
999
		if (args->quiet || args->no_progress)
1000
			strvec_push(&cmd.args, "-q");
1001
		args->check_self_contained_and_connected = 0;
1002
	}
1003

1004
	if (pass_header)
1005
		strvec_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
1006
			     ntohl(header.hdr_version),
1007
				 ntohl(header.hdr_entries));
1008
	if (fsck_objects) {
1009
		if (args->from_promisor || index_pack_args)
1010
			/*
1011
			 * We cannot use --strict in index-pack because it
1012
			 * checks both broken objects and links, but we only
1013
			 * want to check for broken objects.
1014
			 */
1015
			strvec_push(&cmd.args, "--fsck-objects");
1016
		else
1017
			strvec_pushf(&cmd.args, "--strict%s",
1018
				     fsck_msg_types.buf);
1019
	}
1020

1021
	if (index_pack_args) {
1022
		int i;
1023

1024
		for (i = 0; i < cmd.args.nr; i++)
1025
			strvec_push(index_pack_args, cmd.args.v[i]);
1026
	}
1027

1028
	sigchain_push(SIGPIPE, SIG_IGN);
1029

1030
	cmd.in = demux.out;
1031
	cmd.git_cmd = 1;
1032
	if (start_command(&cmd))
1033
		die(_("fetch-pack: unable to fork off %s"), cmd_name);
1034
	if (do_keep && (pack_lockfiles || fsck_objects)) {
1035
		int is_well_formed;
1036
		char *pack_lockfile = index_pack_lockfile(cmd.out, &is_well_formed);
1037

1038
		if (!is_well_formed)
1039
			die(_("fetch-pack: invalid index-pack output"));
1040
		if (pack_lockfiles && pack_lockfile)
1041
			string_list_append_nodup(pack_lockfiles, pack_lockfile);
1042
		else
1043
			free(pack_lockfile);
1044
		parse_gitmodules_oids(cmd.out, gitmodules_oids);
1045
		close(cmd.out);
1046
	}
1047

1048
	if (!use_sideband)
1049
		/* Closed by start_command() */
1050
		xd[0] = -1;
1051

1052
	ret = finish_command(&cmd);
1053
	if (!ret || (args->check_self_contained_and_connected && ret == 1))
1054
		args->self_contained_and_connected =
1055
			args->check_self_contained_and_connected &&
1056
			ret == 0;
1057
	else
1058
		die(_("%s failed"), cmd_name);
1059
	if (use_sideband && finish_async(&demux))
1060
		die(_("error in sideband demultiplexer"));
1061

1062
	sigchain_pop(SIGPIPE);
1063

1064
	/*
1065
	 * Now that index-pack has succeeded, write the promisor file using the
1066
	 * obtained .keep filename if necessary
1067
	 */
1068
	if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
1069
		create_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
1070

1071
	return 0;
1072
}
1073

1074
static int ref_compare_name(const struct ref *a, const struct ref *b)
1075
{
1076
	return strcmp(a->name, b->name);
1077
}
1078

1079
DEFINE_LIST_SORT(static, sort_ref_list, struct ref, next);
1080

1081
static int cmp_ref_by_name(const void *a_, const void *b_)
1082
{
1083
	const struct ref *a = *((const struct ref **)a_);
1084
	const struct ref *b = *((const struct ref **)b_);
1085
	return strcmp(a->name, b->name);
1086
}
1087

1088
static struct ref *do_fetch_pack(struct fetch_pack_args *args,
1089
				 int fd[2],
1090
				 const struct ref *orig_ref,
1091
				 struct ref **sought, int nr_sought,
1092
				 struct shallow_info *si,
1093
				 struct string_list *pack_lockfiles)
1094
{
1095
	struct repository *r = the_repository;
1096
	struct ref *ref = copy_ref_list(orig_ref);
1097
	struct object_id oid;
1098
	const char *agent_feature;
1099
	size_t agent_len;
1100
	struct fetch_negotiator negotiator_alloc;
1101
	struct fetch_negotiator *negotiator;
1102

1103
	negotiator = &negotiator_alloc;
1104
	if (args->refetch) {
1105
		fetch_negotiator_init_noop(negotiator);
1106
	} else {
1107
		fetch_negotiator_init(r, negotiator);
1108
	}
1109

1110
	sort_ref_list(&ref, ref_compare_name);
1111
	QSORT(sought, nr_sought, cmp_ref_by_name);
1112

1113
	if ((agent_feature = server_feature_value("agent", &agent_len))) {
1114
		agent_supported = 1;
1115
		if (agent_len)
1116
			print_verbose(args, _("Server version is %.*s"),
1117
				      (int)agent_len, agent_feature);
1118
	}
1119

1120
	if (!server_supports("session-id"))
1121
		advertise_sid = 0;
1122

1123
	if (server_supports("shallow"))
1124
		print_verbose(args, _("Server supports %s"), "shallow");
1125
	else if (args->depth > 0 || is_repository_shallow(r))
1126
		die(_("Server does not support shallow clients"));
1127
	if (args->depth > 0 || args->deepen_since || args->deepen_not)
1128
		args->deepen = 1;
1129
	if (server_supports("multi_ack_detailed")) {
1130
		print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
1131
		multi_ack = 2;
1132
		if (server_supports("no-done")) {
1133
			print_verbose(args, _("Server supports %s"), "no-done");
1134
			if (args->stateless_rpc)
1135
				no_done = 1;
1136
		}
1137
	}
1138
	else if (server_supports("multi_ack")) {
1139
		print_verbose(args, _("Server supports %s"), "multi_ack");
1140
		multi_ack = 1;
1141
	}
1142
	if (server_supports("side-band-64k")) {
1143
		print_verbose(args, _("Server supports %s"), "side-band-64k");
1144
		use_sideband = 2;
1145
	}
1146
	else if (server_supports("side-band")) {
1147
		print_verbose(args, _("Server supports %s"), "side-band");
1148
		use_sideband = 1;
1149
	}
1150
	if (server_supports("allow-tip-sha1-in-want")) {
1151
		print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
1152
		allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1153
	}
1154
	if (server_supports("allow-reachable-sha1-in-want")) {
1155
		print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
1156
		allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1157
	}
1158
	if (server_supports("thin-pack"))
1159
		print_verbose(args, _("Server supports %s"), "thin-pack");
1160
	else
1161
		args->use_thin_pack = 0;
1162
	if (server_supports("no-progress"))
1163
		print_verbose(args, _("Server supports %s"), "no-progress");
1164
	else
1165
		args->no_progress = 0;
1166
	if (server_supports("include-tag"))
1167
		print_verbose(args, _("Server supports %s"), "include-tag");
1168
	else
1169
		args->include_tag = 0;
1170
	if (server_supports("ofs-delta"))
1171
		print_verbose(args, _("Server supports %s"), "ofs-delta");
1172
	else
1173
		prefer_ofs_delta = 0;
1174

1175
	if (server_supports("filter")) {
1176
		server_supports_filtering = 1;
1177
		print_verbose(args, _("Server supports %s"), "filter");
1178
	} else if (args->filter_options.choice) {
1179
		warning("filtering not recognized by server, ignoring");
1180
	}
1181

1182
	if (server_supports("deepen-since")) {
1183
		print_verbose(args, _("Server supports %s"), "deepen-since");
1184
		deepen_since_ok = 1;
1185
	} else if (args->deepen_since)
1186
		die(_("Server does not support --shallow-since"));
1187
	if (server_supports("deepen-not")) {
1188
		print_verbose(args, _("Server supports %s"), "deepen-not");
1189
		deepen_not_ok = 1;
1190
	} else if (args->deepen_not)
1191
		die(_("Server does not support --shallow-exclude"));
1192
	if (server_supports("deepen-relative"))
1193
		print_verbose(args, _("Server supports %s"), "deepen-relative");
1194
	else if (args->deepen_relative)
1195
		die(_("Server does not support --deepen"));
1196
	if (!server_supports_hash(the_hash_algo->name, NULL))
1197
		die(_("Server does not support this repository's object format"));
1198

1199
	mark_complete_and_common_ref(negotiator, args, &ref);
1200
	filter_refs(args, &ref, sought, nr_sought);
1201
	if (!args->refetch && everything_local(args, &ref)) {
1202
		packet_flush(fd[1]);
1203
		goto all_done;
1204
	}
1205
	if (find_common(negotiator, args, fd, &oid, ref) < 0)
1206
		if (!args->keep_pack)
1207
			/* When cloning, it is not unusual to have
1208
			 * no common commit.
1209
			 */
1210
			warning(_("no common commits"));
1211

1212
	if (args->stateless_rpc)
1213
		packet_flush(fd[1]);
1214
	if (args->deepen)
1215
		setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1216
					NULL);
1217
	else if (si->nr_ours || si->nr_theirs) {
1218
		if (args->reject_shallow_remote)
1219
			die(_("source repository is shallow, reject to clone."));
1220
		alternate_shallow_file = setup_temporary_shallow(si->shallow);
1221
	} else
1222
		alternate_shallow_file = NULL;
1223
	if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
1224
		     &fsck_options.gitmodules_found))
1225
		die(_("git fetch-pack: fetch failed."));
1226
	if (fsck_finish(&fsck_options))
1227
		die("fsck failed");
1228

1229
 all_done:
1230
	if (negotiator)
1231
		negotiator->release(negotiator);
1232
	return ref;
1233
}
1234

1235
static void add_shallow_requests(struct strbuf *req_buf,
1236
				 const struct fetch_pack_args *args)
1237
{
1238
	if (is_repository_shallow(the_repository))
1239
		write_shallow_commits(req_buf, 1, NULL);
1240
	if (args->depth > 0)
1241
		packet_buf_write(req_buf, "deepen %d", args->depth);
1242
	if (args->deepen_since) {
1243
		timestamp_t max_age = approxidate(args->deepen_since);
1244
		packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1245
	}
1246
	if (args->deepen_not) {
1247
		int i;
1248
		for (i = 0; i < args->deepen_not->nr; i++) {
1249
			struct string_list_item *s = args->deepen_not->items + i;
1250
			packet_buf_write(req_buf, "deepen-not %s", s->string);
1251
		}
1252
	}
1253
	if (args->deepen_relative)
1254
		packet_buf_write(req_buf, "deepen-relative\n");
1255
}
1256

1257
static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1258
{
1259
	int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1260

1261
	for ( ; wants ; wants = wants->next) {
1262
		const struct object_id *remote = &wants->old_oid;
1263
		struct object *o;
1264

1265
		/*
1266
		 * If that object is complete (i.e. it is an ancestor of a
1267
		 * local ref), we tell them we have it but do not have to
1268
		 * tell them about its ancestors, which they already know
1269
		 * about.
1270
		 *
1271
		 * We use lookup_object here because we are only
1272
		 * interested in the case we *know* the object is
1273
		 * reachable and we have already scanned it.
1274
		 */
1275
		if (((o = lookup_object(the_repository, remote)) != NULL) &&
1276
		    (o->flags & COMPLETE)) {
1277
			continue;
1278
		}
1279

1280
		if (!use_ref_in_want || wants->exact_oid)
1281
			packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1282
		else
1283
			packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1284
	}
1285
}
1286

1287
static void add_common(struct strbuf *req_buf, struct oidset *common)
1288
{
1289
	struct oidset_iter iter;
1290
	const struct object_id *oid;
1291
	oidset_iter_init(common, &iter);
1292

1293
	while ((oid = oidset_iter_next(&iter))) {
1294
		packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1295
	}
1296
}
1297

1298
static int add_haves(struct fetch_negotiator *negotiator,
1299
		     struct strbuf *req_buf,
1300
		     int *haves_to_send)
1301
{
1302
	int haves_added = 0;
1303
	const struct object_id *oid;
1304

1305
	while ((oid = negotiator->next(negotiator))) {
1306
		packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1307
		if (++haves_added >= *haves_to_send)
1308
			break;
1309
	}
1310

1311
	/* Increase haves to send on next round */
1312
	*haves_to_send = next_flush(1, *haves_to_send);
1313

1314
	return haves_added;
1315
}
1316

1317
static void write_fetch_command_and_capabilities(struct strbuf *req_buf,
1318
						 const struct string_list *server_options)
1319
{
1320
	const char *hash_name;
1321

1322
	ensure_server_supports_v2("fetch");
1323
	packet_buf_write(req_buf, "command=fetch");
1324
	if (server_supports_v2("agent"))
1325
		packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized());
1326
	if (advertise_sid && server_supports_v2("session-id"))
1327
		packet_buf_write(req_buf, "session-id=%s", trace2_session_id());
1328
	if (server_options && server_options->nr) {
1329
		int i;
1330
		ensure_server_supports_v2("server-option");
1331
		for (i = 0; i < server_options->nr; i++)
1332
			packet_buf_write(req_buf, "server-option=%s",
1333
					 server_options->items[i].string);
1334
	}
1335

1336
	if (server_feature_v2("object-format", &hash_name)) {
1337
		int hash_algo = hash_algo_by_name(hash_name);
1338
		if (hash_algo_by_ptr(the_hash_algo) != hash_algo)
1339
			die(_("mismatched algorithms: client %s; server %s"),
1340
			    the_hash_algo->name, hash_name);
1341
		packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name);
1342
	} else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1) {
1343
		die(_("the server does not support algorithm '%s'"),
1344
		    the_hash_algo->name);
1345
	}
1346
	packet_buf_delim(req_buf);
1347
}
1348

1349
static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1350
			      struct fetch_pack_args *args,
1351
			      const struct ref *wants, struct oidset *common,
1352
			      int *haves_to_send, int *in_vain,
1353
			      int sideband_all, int seen_ack)
1354
{
1355
	int haves_added;
1356
	int done_sent = 0;
1357
	struct strbuf req_buf = STRBUF_INIT;
1358

1359
	write_fetch_command_and_capabilities(&req_buf, args->server_options);
1360

1361
	if (args->use_thin_pack)
1362
		packet_buf_write(&req_buf, "thin-pack");
1363
	if (args->no_progress)
1364
		packet_buf_write(&req_buf, "no-progress");
1365
	if (args->include_tag)
1366
		packet_buf_write(&req_buf, "include-tag");
1367
	if (prefer_ofs_delta)
1368
		packet_buf_write(&req_buf, "ofs-delta");
1369
	if (sideband_all)
1370
		packet_buf_write(&req_buf, "sideband-all");
1371

1372
	/* Add shallow-info and deepen request */
1373
	if (server_supports_feature("fetch", "shallow", 0))
1374
		add_shallow_requests(&req_buf, args);
1375
	else if (is_repository_shallow(the_repository) || args->deepen)
1376
		die(_("Server does not support shallow requests"));
1377

1378
	/* Add filter */
1379
	send_filter(args, &req_buf,
1380
		    server_supports_feature("fetch", "filter", 0));
1381

1382
	if (server_supports_feature("fetch", "packfile-uris", 0)) {
1383
		int i;
1384
		struct strbuf to_send = STRBUF_INIT;
1385

1386
		for (i = 0; i < uri_protocols.nr; i++) {
1387
			const char *s = uri_protocols.items[i].string;
1388

1389
			if (!strcmp(s, "https") || !strcmp(s, "http")) {
1390
				if (to_send.len)
1391
					strbuf_addch(&to_send, ',');
1392
				strbuf_addstr(&to_send, s);
1393
			}
1394
		}
1395
		if (to_send.len) {
1396
			packet_buf_write(&req_buf, "packfile-uris %s",
1397
					 to_send.buf);
1398
			strbuf_release(&to_send);
1399
		}
1400
	}
1401

1402
	/* add wants */
1403
	add_wants(wants, &req_buf);
1404

1405
	/* Add all of the common commits we've found in previous rounds */
1406
	add_common(&req_buf, common);
1407

1408
	haves_added = add_haves(negotiator, &req_buf, haves_to_send);
1409
	*in_vain += haves_added;
1410
	trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added);
1411
	trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain);
1412
	if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) {
1413
		/* Send Done */
1414
		packet_buf_write(&req_buf, "done\n");
1415
		done_sent = 1;
1416
	}
1417

1418
	/* Send request */
1419
	packet_buf_flush(&req_buf);
1420
	if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
1421
		die_errno(_("unable to write request to remote"));
1422

1423
	strbuf_release(&req_buf);
1424
	return done_sent;
1425
}
1426

1427
/*
1428
 * Processes a section header in a server's response and checks if it matches
1429
 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1430
 * not consumed); if 0, the line will be consumed and the function will die if
1431
 * the section header doesn't match what was expected.
1432
 */
1433
static int process_section_header(struct packet_reader *reader,
1434
				  const char *section, int peek)
1435
{
1436
	int ret = 0;
1437

1438
	if (packet_reader_peek(reader) == PACKET_READ_NORMAL &&
1439
	    !strcmp(reader->line, section))
1440
		ret = 1;
1441

1442
	if (!peek) {
1443
		if (!ret) {
1444
			if (reader->line)
1445
				die(_("expected '%s', received '%s'"),
1446
				    section, reader->line);
1447
			else
1448
				die(_("expected '%s'"), section);
1449
		}
1450
		packet_reader_read(reader);
1451
	}
1452

1453
	return ret;
1454
}
1455

1456
static int process_ack(struct fetch_negotiator *negotiator,
1457
		       struct packet_reader *reader,
1458
		       struct object_id *common_oid,
1459
		       int *received_ready)
1460
{
1461
	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1462
		const char *arg;
1463

1464
		if (!strcmp(reader->line, "NAK"))
1465
			continue;
1466

1467
		if (skip_prefix(reader->line, "ACK ", &arg)) {
1468
			if (!get_oid_hex(arg, common_oid)) {
1469
				struct commit *commit;
1470
				commit = lookup_commit(the_repository, common_oid);
1471
				if (negotiator)
1472
					negotiator->ack(negotiator, commit);
1473
			}
1474
			return 1;
1475
		}
1476

1477
		if (!strcmp(reader->line, "ready")) {
1478
			*received_ready = 1;
1479
			continue;
1480
		}
1481

1482
		die(_("unexpected acknowledgment line: '%s'"), reader->line);
1483
	}
1484

1485
	if (reader->status != PACKET_READ_FLUSH &&
1486
	    reader->status != PACKET_READ_DELIM)
1487
		die(_("error processing acks: %d"), reader->status);
1488

1489
	/*
1490
	 * If an "acknowledgments" section is sent, a packfile is sent if and
1491
	 * only if "ready" was sent in this section. The other sections
1492
	 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1493
	 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1494
	 * otherwise.
1495
	 */
1496
	if (*received_ready && reader->status != PACKET_READ_DELIM)
1497
		/*
1498
		 * TRANSLATORS: The parameter will be 'ready', a protocol
1499
		 * keyword.
1500
		 */
1501
		die(_("expected packfile to be sent after '%s'"), "ready");
1502
	if (!*received_ready && reader->status != PACKET_READ_FLUSH)
1503
		/*
1504
		 * TRANSLATORS: The parameter will be 'ready', a protocol
1505
		 * keyword.
1506
		 */
1507
		die(_("expected no other sections to be sent after no '%s'"), "ready");
1508

1509
	return 0;
1510
}
1511

1512
static void receive_shallow_info(struct fetch_pack_args *args,
1513
				 struct packet_reader *reader,
1514
				 struct oid_array *shallows,
1515
				 struct shallow_info *si)
1516
{
1517
	int unshallow_received = 0;
1518

1519
	process_section_header(reader, "shallow-info", 0);
1520
	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1521
		const char *arg;
1522
		struct object_id oid;
1523

1524
		if (skip_prefix(reader->line, "shallow ", &arg)) {
1525
			if (get_oid_hex(arg, &oid))
1526
				die(_("invalid shallow line: %s"), reader->line);
1527
			oid_array_append(shallows, &oid);
1528
			continue;
1529
		}
1530
		if (skip_prefix(reader->line, "unshallow ", &arg)) {
1531
			if (get_oid_hex(arg, &oid))
1532
				die(_("invalid unshallow line: %s"), reader->line);
1533
			if (!lookup_object(the_repository, &oid))
1534
				die(_("object not found: %s"), reader->line);
1535
			/* make sure that it is parsed as shallow */
1536
			if (!parse_object(the_repository, &oid))
1537
				die(_("error in object: %s"), reader->line);
1538
			if (unregister_shallow(&oid))
1539
				die(_("no shallow found: %s"), reader->line);
1540
			unshallow_received = 1;
1541
			continue;
1542
		}
1543
		die(_("expected shallow/unshallow, got %s"), reader->line);
1544
	}
1545

1546
	if (reader->status != PACKET_READ_FLUSH &&
1547
	    reader->status != PACKET_READ_DELIM)
1548
		die(_("error processing shallow info: %d"), reader->status);
1549

1550
	if (args->deepen || unshallow_received) {
1551
		/*
1552
		 * Treat these as shallow lines caused by our depth settings.
1553
		 * In v0, these lines cannot cause refs to be rejected; do the
1554
		 * same.
1555
		 */
1556
		int i;
1557

1558
		for (i = 0; i < shallows->nr; i++)
1559
			register_shallow(the_repository, &shallows->oid[i]);
1560
		setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1561
					NULL);
1562
		args->deepen = 1;
1563
	} else if (shallows->nr) {
1564
		/*
1565
		 * Treat these as shallow lines caused by the remote being
1566
		 * shallow. In v0, remote refs that reach these objects are
1567
		 * rejected (unless --update-shallow is set); do the same.
1568
		 */
1569
		prepare_shallow_info(si, shallows);
1570
		if (si->nr_ours || si->nr_theirs) {
1571
			if (args->reject_shallow_remote)
1572
				die(_("source repository is shallow, reject to clone."));
1573
			alternate_shallow_file =
1574
				setup_temporary_shallow(si->shallow);
1575
		} else
1576
			alternate_shallow_file = NULL;
1577
	} else {
1578
		alternate_shallow_file = NULL;
1579
	}
1580
}
1581

1582
static int cmp_name_ref(const void *name, const void *ref)
1583
{
1584
	return strcmp(name, (*(struct ref **)ref)->name);
1585
}
1586

1587
static void receive_wanted_refs(struct packet_reader *reader,
1588
				struct ref **sought, int nr_sought)
1589
{
1590
	process_section_header(reader, "wanted-refs", 0);
1591
	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1592
		struct object_id oid;
1593
		const char *end;
1594
		struct ref **found;
1595

1596
		if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1597
			die(_("expected wanted-ref, got '%s'"), reader->line);
1598

1599
		found = bsearch(end, sought, nr_sought, sizeof(*sought),
1600
				cmp_name_ref);
1601
		if (!found)
1602
			die(_("unexpected wanted-ref: '%s'"), reader->line);
1603
		oidcpy(&(*found)->old_oid, &oid);
1604
	}
1605

1606
	if (reader->status != PACKET_READ_DELIM)
1607
		die(_("error processing wanted refs: %d"), reader->status);
1608
}
1609

1610
static void receive_packfile_uris(struct packet_reader *reader,
1611
				  struct string_list *uris)
1612
{
1613
	process_section_header(reader, "packfile-uris", 0);
1614
	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1615
		if (reader->pktlen < the_hash_algo->hexsz ||
1616
		    reader->line[the_hash_algo->hexsz] != ' ')
1617
			die("expected '<hash> <uri>', got: %s\n", reader->line);
1618

1619
		string_list_append(uris, reader->line);
1620
	}
1621
	if (reader->status != PACKET_READ_DELIM)
1622
		die("expected DELIM");
1623
}
1624

1625
enum fetch_state {
1626
	FETCH_CHECK_LOCAL = 0,
1627
	FETCH_SEND_REQUEST,
1628
	FETCH_PROCESS_ACKS,
1629
	FETCH_GET_PACK,
1630
	FETCH_DONE,
1631
};
1632

1633
static void do_check_stateless_delimiter(int stateless_rpc,
1634
					 struct packet_reader *reader)
1635
{
1636
	check_stateless_delimiter(stateless_rpc, reader,
1637
				  _("git fetch-pack: expected response end packet"));
1638
}
1639

1640
static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1641
				    int fd[2],
1642
				    const struct ref *orig_ref,
1643
				    struct ref **sought, int nr_sought,
1644
				    struct oid_array *shallows,
1645
				    struct shallow_info *si,
1646
				    struct string_list *pack_lockfiles)
1647
{
1648
	struct repository *r = the_repository;
1649
	struct ref *ref = copy_ref_list(orig_ref);
1650
	enum fetch_state state = FETCH_CHECK_LOCAL;
1651
	struct oidset common = OIDSET_INIT;
1652
	struct packet_reader reader;
1653
	int in_vain = 0, negotiation_started = 0;
1654
	int negotiation_round = 0;
1655
	int haves_to_send = INITIAL_FLUSH;
1656
	struct fetch_negotiator negotiator_alloc;
1657
	struct fetch_negotiator *negotiator;
1658
	int seen_ack = 0;
1659
	struct object_id common_oid;
1660
	int received_ready = 0;
1661
	struct string_list packfile_uris = STRING_LIST_INIT_DUP;
1662
	int i;
1663
	struct strvec index_pack_args = STRVEC_INIT;
1664

1665
	negotiator = &negotiator_alloc;
1666
	if (args->refetch)
1667
		fetch_negotiator_init_noop(negotiator);
1668
	else
1669
		fetch_negotiator_init(r, negotiator);
1670

1671
	packet_reader_init(&reader, fd[0], NULL, 0,
1672
			   PACKET_READ_CHOMP_NEWLINE |
1673
			   PACKET_READ_DIE_ON_ERR_PACKET);
1674
	if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1675
	    server_supports_feature("fetch", "sideband-all", 0)) {
1676
		reader.use_sideband = 1;
1677
		reader.me = "fetch-pack";
1678
	}
1679

1680
	while (state != FETCH_DONE) {
1681
		switch (state) {
1682
		case FETCH_CHECK_LOCAL:
1683
			sort_ref_list(&ref, ref_compare_name);
1684
			QSORT(sought, nr_sought, cmp_ref_by_name);
1685

1686
			/* v2 supports these by default */
1687
			allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1688
			use_sideband = 2;
1689
			if (args->depth > 0 || args->deepen_since || args->deepen_not)
1690
				args->deepen = 1;
1691

1692
			/* Filter 'ref' by 'sought' and those that aren't local */
1693
			mark_complete_and_common_ref(negotiator, args, &ref);
1694
			filter_refs(args, &ref, sought, nr_sought);
1695
			if (!args->refetch && everything_local(args, &ref))
1696
				state = FETCH_DONE;
1697
			else
1698
				state = FETCH_SEND_REQUEST;
1699

1700
			mark_tips(negotiator, args->negotiation_tips);
1701
			for_each_cached_alternate(negotiator,
1702
						  insert_one_alternate_object);
1703
			break;
1704
		case FETCH_SEND_REQUEST:
1705
			if (!negotiation_started) {
1706
				negotiation_started = 1;
1707
				trace2_region_enter("fetch-pack",
1708
						    "negotiation_v2",
1709
						    the_repository);
1710
			}
1711
			negotiation_round++;
1712
			trace2_region_enter_printf("negotiation_v2", "round",
1713
						   the_repository, "%d",
1714
						   negotiation_round);
1715
			if (send_fetch_request(negotiator, fd[1], args, ref,
1716
					       &common,
1717
					       &haves_to_send, &in_vain,
1718
					       reader.use_sideband,
1719
					       seen_ack)) {
1720
				trace2_region_leave_printf("negotiation_v2", "round",
1721
							   the_repository, "%d",
1722
							   negotiation_round);
1723
				state = FETCH_GET_PACK;
1724
			}
1725
			else
1726
				state = FETCH_PROCESS_ACKS;
1727
			break;
1728
		case FETCH_PROCESS_ACKS:
1729
			/* Process ACKs/NAKs */
1730
			process_section_header(&reader, "acknowledgments", 0);
1731
			while (process_ack(negotiator, &reader, &common_oid,
1732
					   &received_ready)) {
1733
				in_vain = 0;
1734
				seen_ack = 1;
1735
				oidset_insert(&common, &common_oid);
1736
			}
1737
			trace2_region_leave_printf("negotiation_v2", "round",
1738
						   the_repository, "%d",
1739
						   negotiation_round);
1740
			if (received_ready) {
1741
				/*
1742
				 * Don't check for response delimiter; get_pack() will
1743
				 * read the rest of this response.
1744
				 */
1745
				state = FETCH_GET_PACK;
1746
			} else {
1747
				do_check_stateless_delimiter(args->stateless_rpc, &reader);
1748
				state = FETCH_SEND_REQUEST;
1749
			}
1750
			break;
1751
		case FETCH_GET_PACK:
1752
			trace2_region_leave("fetch-pack",
1753
					    "negotiation_v2",
1754
					    the_repository);
1755
			trace2_data_intmax("negotiation_v2", the_repository,
1756
					   "total_rounds", negotiation_round);
1757
			/* Check for shallow-info section */
1758
			if (process_section_header(&reader, "shallow-info", 1))
1759
				receive_shallow_info(args, &reader, shallows, si);
1760

1761
			if (process_section_header(&reader, "wanted-refs", 1))
1762
				receive_wanted_refs(&reader, sought, nr_sought);
1763

1764
			/* get the pack(s) */
1765
			if (git_env_bool("GIT_TRACE_REDACT", 1))
1766
				reader.options |= PACKET_READ_REDACT_URI_PATH;
1767
			if (process_section_header(&reader, "packfile-uris", 1))
1768
				receive_packfile_uris(&reader, &packfile_uris);
1769
			/* We don't expect more URIs. Reset to avoid expensive URI check. */
1770
			reader.options &= ~PACKET_READ_REDACT_URI_PATH;
1771

1772
			process_section_header(&reader, "packfile", 0);
1773

1774
			/*
1775
			 * this is the final request we'll make of the server;
1776
			 * do a half-duplex shutdown to indicate that they can
1777
			 * hang up as soon as the pack is sent.
1778
			 */
1779
			close(fd[1]);
1780
			fd[1] = -1;
1781

1782
			if (get_pack(args, fd, pack_lockfiles,
1783
				     packfile_uris.nr ? &index_pack_args : NULL,
1784
				     sought, nr_sought, &fsck_options.gitmodules_found))
1785
				die(_("git fetch-pack: fetch failed."));
1786
			do_check_stateless_delimiter(args->stateless_rpc, &reader);
1787

1788
			state = FETCH_DONE;
1789
			break;
1790
		case FETCH_DONE:
1791
			continue;
1792
		}
1793
	}
1794

1795
	for (i = 0; i < packfile_uris.nr; i++) {
1796
		int j;
1797
		struct child_process cmd = CHILD_PROCESS_INIT;
1798
		char packname[GIT_MAX_HEXSZ + 1];
1799
		const char *uri = packfile_uris.items[i].string +
1800
			the_hash_algo->hexsz + 1;
1801

1802
		strvec_push(&cmd.args, "http-fetch");
1803
		strvec_pushf(&cmd.args, "--packfile=%.*s",
1804
			     (int) the_hash_algo->hexsz,
1805
			     packfile_uris.items[i].string);
1806
		for (j = 0; j < index_pack_args.nr; j++)
1807
			strvec_pushf(&cmd.args, "--index-pack-arg=%s",
1808
				     index_pack_args.v[j]);
1809
		strvec_push(&cmd.args, uri);
1810
		cmd.git_cmd = 1;
1811
		cmd.no_stdin = 1;
1812
		cmd.out = -1;
1813
		if (start_command(&cmd))
1814
			die("fetch-pack: unable to spawn http-fetch");
1815

1816
		if (read_in_full(cmd.out, packname, 5) < 0 ||
1817
		    memcmp(packname, "keep\t", 5))
1818
			die("fetch-pack: expected keep then TAB at start of http-fetch output");
1819

1820
		if (read_in_full(cmd.out, packname,
1821
				 the_hash_algo->hexsz + 1) < 0 ||
1822
		    packname[the_hash_algo->hexsz] != '\n')
1823
			die("fetch-pack: expected hash then LF at end of http-fetch output");
1824

1825
		packname[the_hash_algo->hexsz] = '\0';
1826

1827
		parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
1828

1829
		close(cmd.out);
1830

1831
		if (finish_command(&cmd))
1832
			die("fetch-pack: unable to finish http-fetch");
1833

1834
		if (memcmp(packfile_uris.items[i].string, packname,
1835
			   the_hash_algo->hexsz))
1836
			die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1837
			    uri, (int) the_hash_algo->hexsz,
1838
			    packfile_uris.items[i].string);
1839

1840
		string_list_append_nodup(pack_lockfiles,
1841
					 xstrfmt("%s/pack/pack-%s.keep",
1842
						 get_object_directory(),
1843
						 packname));
1844
	}
1845
	string_list_clear(&packfile_uris, 0);
1846
	strvec_clear(&index_pack_args);
1847

1848
	if (fsck_finish(&fsck_options))
1849
		die("fsck failed");
1850

1851
	if (negotiator)
1852
		negotiator->release(negotiator);
1853

1854
	oidset_clear(&common);
1855
	return ref;
1856
}
1857

1858
static int fetch_pack_config_cb(const char *var, const char *value,
1859
				const struct config_context *ctx, void *cb)
1860
{
1861
	const char *msg_id;
1862

1863
	if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1864
		char *path ;
1865

1866
		if (git_config_pathname(&path, var, value))
1867
			return 1;
1868
		strbuf_addf(&fsck_msg_types, "%cskiplist=%s",
1869
			fsck_msg_types.len ? ',' : '=', path);
1870
		free(path);
1871
		return 0;
1872
	}
1873

1874
	if (skip_prefix(var, "fetch.fsck.", &msg_id)) {
1875
		if (!value)
1876
			return config_error_nonbool(var);
1877
		if (is_valid_msg_type(msg_id, value))
1878
			strbuf_addf(&fsck_msg_types, "%c%s=%s",
1879
				fsck_msg_types.len ? ',' : '=', msg_id, value);
1880
		else
1881
			warning("Skipping unknown msg id '%s'", msg_id);
1882
		return 0;
1883
	}
1884

1885
	return git_default_config(var, value, ctx, cb);
1886
}
1887

1888
static void fetch_pack_config(void)
1889
{
1890
	git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1891
	git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1892
	git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1893
	git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1894
	git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1895
	git_config_get_bool("transfer.advertisesid", &advertise_sid);
1896
	if (!uri_protocols.nr) {
1897
		char *str;
1898

1899
		if (!git_config_get_string("fetch.uriprotocols", &str) && str) {
1900
			string_list_split(&uri_protocols, str, ',', -1);
1901
			free(str);
1902
		}
1903
	}
1904

1905
	git_config(fetch_pack_config_cb, NULL);
1906
}
1907

1908
static void fetch_pack_setup(void)
1909
{
1910
	static int did_setup;
1911
	if (did_setup)
1912
		return;
1913
	fetch_pack_config();
1914
	if (0 <= fetch_unpack_limit)
1915
		unpack_limit = fetch_unpack_limit;
1916
	else if (0 <= transfer_unpack_limit)
1917
		unpack_limit = transfer_unpack_limit;
1918
	did_setup = 1;
1919
}
1920

1921
static int remove_duplicates_in_refs(struct ref **ref, int nr)
1922
{
1923
	struct string_list names = STRING_LIST_INIT_NODUP;
1924
	int src, dst;
1925

1926
	for (src = dst = 0; src < nr; src++) {
1927
		struct string_list_item *item;
1928
		item = string_list_insert(&names, ref[src]->name);
1929
		if (item->util)
1930
			continue; /* already have it */
1931
		item->util = ref[src];
1932
		if (src != dst)
1933
			ref[dst] = ref[src];
1934
		dst++;
1935
	}
1936
	for (src = dst; src < nr; src++)
1937
		ref[src] = NULL;
1938
	string_list_clear(&names, 0);
1939
	return dst;
1940
}
1941

1942
static void update_shallow(struct fetch_pack_args *args,
1943
			   struct ref **sought, int nr_sought,
1944
			   struct shallow_info *si)
1945
{
1946
	struct oid_array ref = OID_ARRAY_INIT;
1947
	int *status;
1948
	int i;
1949

1950
	if (args->deepen && alternate_shallow_file) {
1951
		if (*alternate_shallow_file == '\0') { /* --unshallow */
1952
			unlink_or_warn(git_path_shallow(the_repository));
1953
			rollback_shallow_file(the_repository, &shallow_lock);
1954
		} else
1955
			commit_shallow_file(the_repository, &shallow_lock);
1956
		alternate_shallow_file = NULL;
1957
		return;
1958
	}
1959

1960
	if (!si->shallow || !si->shallow->nr)
1961
		return;
1962

1963
	if (args->cloning) {
1964
		/*
1965
		 * remote is shallow, but this is a clone, there are
1966
		 * no objects in repo to worry about. Accept any
1967
		 * shallow points that exist in the pack (iow in repo
1968
		 * after get_pack() and reprepare_packed_git())
1969
		 */
1970
		struct oid_array extra = OID_ARRAY_INIT;
1971
		struct object_id *oid = si->shallow->oid;
1972
		for (i = 0; i < si->shallow->nr; i++)
1973
			if (repo_has_object_file(the_repository, &oid[i]))
1974
				oid_array_append(&extra, &oid[i]);
1975
		if (extra.nr) {
1976
			setup_alternate_shallow(&shallow_lock,
1977
						&alternate_shallow_file,
1978
						&extra);
1979
			commit_shallow_file(the_repository, &shallow_lock);
1980
			alternate_shallow_file = NULL;
1981
		}
1982
		oid_array_clear(&extra);
1983
		return;
1984
	}
1985

1986
	if (!si->nr_ours && !si->nr_theirs)
1987
		return;
1988

1989
	remove_nonexistent_theirs_shallow(si);
1990
	if (!si->nr_ours && !si->nr_theirs)
1991
		return;
1992
	for (i = 0; i < nr_sought; i++)
1993
		oid_array_append(&ref, &sought[i]->old_oid);
1994
	si->ref = &ref;
1995

1996
	if (args->update_shallow) {
1997
		/*
1998
		 * remote is also shallow, .git/shallow may be updated
1999
		 * so all refs can be accepted. Make sure we only add
2000
		 * shallow roots that are actually reachable from new
2001
		 * refs.
2002
		 */
2003
		struct oid_array extra = OID_ARRAY_INIT;
2004
		struct object_id *oid = si->shallow->oid;
2005
		assign_shallow_commits_to_refs(si, NULL, NULL);
2006
		if (!si->nr_ours && !si->nr_theirs) {
2007
			oid_array_clear(&ref);
2008
			return;
2009
		}
2010
		for (i = 0; i < si->nr_ours; i++)
2011
			oid_array_append(&extra, &oid[si->ours[i]]);
2012
		for (i = 0; i < si->nr_theirs; i++)
2013
			oid_array_append(&extra, &oid[si->theirs[i]]);
2014
		setup_alternate_shallow(&shallow_lock,
2015
					&alternate_shallow_file,
2016
					&extra);
2017
		commit_shallow_file(the_repository, &shallow_lock);
2018
		oid_array_clear(&extra);
2019
		oid_array_clear(&ref);
2020
		alternate_shallow_file = NULL;
2021
		return;
2022
	}
2023

2024
	/*
2025
	 * remote is also shallow, check what ref is safe to update
2026
	 * without updating .git/shallow
2027
	 */
2028
	CALLOC_ARRAY(status, nr_sought);
2029
	assign_shallow_commits_to_refs(si, NULL, status);
2030
	if (si->nr_ours || si->nr_theirs) {
2031
		for (i = 0; i < nr_sought; i++)
2032
			if (status[i])
2033
				sought[i]->status = REF_STATUS_REJECT_SHALLOW;
2034
	}
2035
	free(status);
2036
	oid_array_clear(&ref);
2037
}
2038

2039
static const struct object_id *iterate_ref_map(void *cb_data)
2040
{
2041
	struct ref **rm = cb_data;
2042
	struct ref *ref = *rm;
2043

2044
	if (!ref)
2045
		return NULL;
2046
	*rm = ref->next;
2047
	return &ref->old_oid;
2048
}
2049

2050
int fetch_pack_fsck_objects(void)
2051
{
2052
	fetch_pack_setup();
2053
	if (fetch_fsck_objects >= 0)
2054
		return fetch_fsck_objects;
2055
	if (transfer_fsck_objects >= 0)
2056
		return transfer_fsck_objects;
2057
	return 0;
2058
}
2059

2060
struct ref *fetch_pack(struct fetch_pack_args *args,
2061
		       int fd[],
2062
		       const struct ref *ref,
2063
		       struct ref **sought, int nr_sought,
2064
		       struct oid_array *shallow,
2065
		       struct string_list *pack_lockfiles,
2066
		       enum protocol_version version)
2067
{
2068
	struct ref *ref_cpy;
2069
	struct shallow_info si;
2070
	struct oid_array shallows_scratch = OID_ARRAY_INIT;
2071

2072
	fetch_pack_setup();
2073
	if (nr_sought)
2074
		nr_sought = remove_duplicates_in_refs(sought, nr_sought);
2075

2076
	if (version != protocol_v2 && !ref) {
2077
		packet_flush(fd[1]);
2078
		die(_("no matching remote head"));
2079
	}
2080
	if (version == protocol_v2) {
2081
		if (shallow->nr)
2082
			BUG("Protocol V2 does not provide shallows at this point in the fetch");
2083
		memset(&si, 0, sizeof(si));
2084
		ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
2085
					   &shallows_scratch, &si,
2086
					   pack_lockfiles);
2087
	} else {
2088
		prepare_shallow_info(&si, shallow);
2089
		ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
2090
					&si, pack_lockfiles);
2091
	}
2092
	reprepare_packed_git(the_repository);
2093

2094
	if (!args->cloning && args->deepen) {
2095
		struct check_connected_options opt = CHECK_CONNECTED_INIT;
2096
		struct ref *iterator = ref_cpy;
2097
		opt.shallow_file = alternate_shallow_file;
2098
		if (args->deepen)
2099
			opt.is_deepening_fetch = 1;
2100
		if (check_connected(iterate_ref_map, &iterator, &opt)) {
2101
			error(_("remote did not send all necessary objects"));
2102
			free_refs(ref_cpy);
2103
			ref_cpy = NULL;
2104
			rollback_shallow_file(the_repository, &shallow_lock);
2105
			goto cleanup;
2106
		}
2107
		args->connectivity_checked = 1;
2108
	}
2109

2110
	update_shallow(args, sought, nr_sought, &si);
2111
cleanup:
2112
	clear_shallow_info(&si);
2113
	oid_array_clear(&shallows_scratch);
2114
	return ref_cpy;
2115
}
2116

2117
static int add_to_object_array(const struct object_id *oid, void *data)
2118
{
2119
	struct object_array *a = data;
2120

2121
	add_object_array(lookup_object(the_repository, oid), "", a);
2122
	return 0;
2123
}
2124

2125
static void clear_common_flag(struct oidset *s)
2126
{
2127
	struct oidset_iter iter;
2128
	const struct object_id *oid;
2129
	oidset_iter_init(s, &iter);
2130

2131
	while ((oid = oidset_iter_next(&iter))) {
2132
		struct object *obj = lookup_object(the_repository, oid);
2133
		obj->flags &= ~COMMON;
2134
	}
2135
}
2136

2137
void negotiate_using_fetch(const struct oid_array *negotiation_tips,
2138
			   const struct string_list *server_options,
2139
			   int stateless_rpc,
2140
			   int fd[],
2141
			   struct oidset *acked_commits)
2142
{
2143
	struct fetch_negotiator negotiator;
2144
	struct packet_reader reader;
2145
	struct object_array nt_object_array = OBJECT_ARRAY_INIT;
2146
	struct strbuf req_buf = STRBUF_INIT;
2147
	int haves_to_send = INITIAL_FLUSH;
2148
	int in_vain = 0;
2149
	int seen_ack = 0;
2150
	int last_iteration = 0;
2151
	int negotiation_round = 0;
2152
	timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
2153

2154
	fetch_negotiator_init(the_repository, &negotiator);
2155
	mark_tips(&negotiator, negotiation_tips);
2156

2157
	packet_reader_init(&reader, fd[0], NULL, 0,
2158
			   PACKET_READ_CHOMP_NEWLINE |
2159
			   PACKET_READ_DIE_ON_ERR_PACKET);
2160

2161
	oid_array_for_each((struct oid_array *) negotiation_tips,
2162
			   add_to_object_array,
2163
			   &nt_object_array);
2164

2165
	trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository);
2166
	while (!last_iteration) {
2167
		int haves_added;
2168
		struct object_id common_oid;
2169
		int received_ready = 0;
2170

2171
		negotiation_round++;
2172

2173
		trace2_region_enter_printf("negotiate_using_fetch", "round",
2174
					   the_repository, "%d",
2175
					   negotiation_round);
2176
		strbuf_reset(&req_buf);
2177
		write_fetch_command_and_capabilities(&req_buf, server_options);
2178

2179
		packet_buf_write(&req_buf, "wait-for-done");
2180

2181
		haves_added = add_haves(&negotiator, &req_buf, &haves_to_send);
2182
		in_vain += haves_added;
2183
		if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN))
2184
			last_iteration = 1;
2185

2186
		trace2_data_intmax("negotiate_using_fetch", the_repository,
2187
				   "haves_added", haves_added);
2188
		trace2_data_intmax("negotiate_using_fetch", the_repository,
2189
				   "in_vain", in_vain);
2190

2191
		/* Send request */
2192
		packet_buf_flush(&req_buf);
2193
		if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0)
2194
			die_errno(_("unable to write request to remote"));
2195

2196
		/* Process ACKs/NAKs */
2197
		process_section_header(&reader, "acknowledgments", 0);
2198
		while (process_ack(&negotiator, &reader, &common_oid,
2199
				   &received_ready)) {
2200
			struct commit *commit = lookup_commit(the_repository,
2201
							      &common_oid);
2202
			if (commit) {
2203
				timestamp_t generation;
2204

2205
				parse_commit_or_die(commit);
2206
				commit->object.flags |= COMMON;
2207
				generation = commit_graph_generation(commit);
2208
				if (generation < min_generation)
2209
					min_generation = generation;
2210
			}
2211
			in_vain = 0;
2212
			seen_ack = 1;
2213
			oidset_insert(acked_commits, &common_oid);
2214
		}
2215
		if (received_ready)
2216
			die(_("unexpected 'ready' from remote"));
2217
		else
2218
			do_check_stateless_delimiter(stateless_rpc, &reader);
2219
		if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
2220
						 REACH_SCRATCH, 0,
2221
						 min_generation))
2222
			last_iteration = 1;
2223
		trace2_region_leave_printf("negotiation", "round",
2224
					   the_repository, "%d",
2225
					   negotiation_round);
2226
	}
2227
	trace2_region_leave("fetch-pack", "negotiate_using_fetch", the_repository);
2228
	trace2_data_intmax("negotiate_using_fetch", the_repository,
2229
			   "total_rounds", negotiation_round);
2230
	clear_common_flag(acked_commits);
2231
	strbuf_release(&req_buf);
2232
}
2233

2234
int report_unmatched_refs(struct ref **sought, int nr_sought)
2235
{
2236
	int i, ret = 0;
2237

2238
	for (i = 0; i < nr_sought; i++) {
2239
		if (!sought[i])
2240
			continue;
2241
		switch (sought[i]->match_status) {
2242
		case REF_MATCHED:
2243
			continue;
2244
		case REF_NOT_MATCHED:
2245
			error(_("no such remote ref %s"), sought[i]->name);
2246
			break;
2247
		case REF_UNADVERTISED_NOT_ALLOWED:
2248
			error(_("Server does not allow request for unadvertised object %s"),
2249
			      sought[i]->name);
2250
			break;
2251
		}
2252
		ret = 1;
2253
	}
2254
	return ret;
2255
}
2256