git

1
#define USE_THE_REPOSITORY_VARIABLE
2

3
#include "git-compat-util.h"
4
#include "config.h"
5
#include "environment.h"
6
#include "gettext.h"
7
#include "hex.h"
8
#include "refs.h"
9
#include "pkt-line.h"
10
#include "sideband.h"
11
#include "repository.h"
12
#include "object-store-ll.h"
13
#include "oid-array.h"
14
#include "object.h"
15
#include "commit.h"
16
#include "diff.h"
17
#include "revision.h"
18
#include "list-objects-filter-options.h"
19
#include "run-command.h"
20
#include "connect.h"
21
#include "sigchain.h"
22
#include "version.h"
23
#include "string-list.h"
24
#include "strvec.h"
25
#include "trace2.h"
26
#include "protocol.h"
27
#include "upload-pack.h"
28
#include "commit-graph.h"
29
#include "commit-reach.h"
30
#include "shallow.h"
31
#include "write-or-die.h"
32
#include "json-writer.h"
33
#include "strmap.h"
34

35
/* Remember to update object flag allocation in object.h */
36
#define THEY_HAVE	(1u << 11)
37
#define OUR_REF		(1u << 12)
38
#define WANTED		(1u << 13)
39
#define COMMON_KNOWN	(1u << 14)
40

41
#define SHALLOW		(1u << 16)
42
#define NOT_SHALLOW	(1u << 17)
43
#define CLIENT_SHALLOW	(1u << 18)
44
#define HIDDEN_REF	(1u << 19)
45

46
#define ALL_FLAGS (THEY_HAVE | OUR_REF | WANTED | COMMON_KNOWN | SHALLOW | \
47
		NOT_SHALLOW | CLIENT_SHALLOW | HIDDEN_REF)
48

49
/* Enum for allowed unadvertised object request (UOR) */
50
enum allow_uor {
51
	/* Allow specifying sha1 if it is a ref tip. */
52
	ALLOW_TIP_SHA1 = 0x01,
53
	/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
54
	ALLOW_REACHABLE_SHA1 = 0x02,
55
	/* Allow request of any sha1. Implies ALLOW_TIP_SHA1 and ALLOW_REACHABLE_SHA1. */
56
	ALLOW_ANY_SHA1 = 0x07
57
};
58

59
/*
60
 * Please annotate, and if possible group together, fields used only
61
 * for protocol v0 or only for protocol v2.
62
 */
63
struct upload_pack_data {
64
	struct string_list symref;				/* v0 only */
65
	struct object_array want_obj;
66
	struct object_array have_obj;
67
	struct strmap wanted_refs;				/* v2 only */
68
	struct strvec hidden_refs;
69

70
	struct object_array shallows;
71
	struct oidset deepen_not;
72
	struct object_array extra_edge_obj;
73
	int depth;
74
	timestamp_t deepen_since;
75
	int deepen_rev_list;
76
	int deepen_relative;
77
	int keepalive;
78
	int shallow_nr;
79
	timestamp_t oldest_have;
80

81
	unsigned int timeout;					/* v0 only */
82
	enum {
83
		NO_MULTI_ACK = 0,
84
		MULTI_ACK = 1,
85
		MULTI_ACK_DETAILED = 2
86
	} multi_ack;						/* v0 only */
87

88
	/* 0 for no sideband, otherwise DEFAULT_PACKET_MAX or LARGE_PACKET_MAX */
89
	int use_sideband;
90

91
	struct string_list uri_protocols;
92
	enum allow_uor allow_uor;
93

94
	struct list_objects_filter_options filter_options;
95
	struct string_list allowed_filters;
96

97
	struct packet_writer writer;
98

99
	char *pack_objects_hook;
100

101
	unsigned stateless_rpc : 1;				/* v0 only */
102
	unsigned no_done : 1;					/* v0 only */
103
	unsigned daemon_mode : 1;				/* v0 only */
104
	unsigned filter_capability_requested : 1;		/* v0 only */
105

106
	unsigned use_thin_pack : 1;
107
	unsigned use_ofs_delta : 1;
108
	unsigned no_progress : 1;
109
	unsigned use_include_tag : 1;
110
	unsigned wait_for_done : 1;
111
	unsigned allow_filter : 1;
112
	unsigned allow_filter_fallback : 1;
113
	unsigned long tree_filter_max_depth;
114

115
	unsigned done : 1;					/* v2 only */
116
	unsigned allow_ref_in_want : 1;				/* v2 only */
117
	unsigned allow_sideband_all : 1;			/* v2 only */
118
	unsigned seen_haves : 1;				/* v2 only */
119
	unsigned allow_packfile_uris : 1;			/* v2 only */
120
	unsigned advertise_sid : 1;
121
	unsigned sent_capabilities : 1;
122
};
123

124
static void upload_pack_data_init(struct upload_pack_data *data)
125
{
126
	struct string_list symref = STRING_LIST_INIT_DUP;
127
	struct strmap wanted_refs = STRMAP_INIT;
128
	struct strvec hidden_refs = STRVEC_INIT;
129
	struct object_array want_obj = OBJECT_ARRAY_INIT;
130
	struct object_array have_obj = OBJECT_ARRAY_INIT;
131
	struct object_array shallows = OBJECT_ARRAY_INIT;
132
	struct oidset deepen_not = OID_ARRAY_INIT;
133
	struct string_list uri_protocols = STRING_LIST_INIT_DUP;
134
	struct object_array extra_edge_obj = OBJECT_ARRAY_INIT;
135
	struct string_list allowed_filters = STRING_LIST_INIT_DUP;
136

137
	memset(data, 0, sizeof(*data));
138
	data->symref = symref;
139
	data->wanted_refs = wanted_refs;
140
	data->hidden_refs = hidden_refs;
141
	data->want_obj = want_obj;
142
	data->have_obj = have_obj;
143
	data->shallows = shallows;
144
	data->deepen_not = deepen_not;
145
	data->uri_protocols = uri_protocols;
146
	data->extra_edge_obj = extra_edge_obj;
147
	data->allowed_filters = allowed_filters;
148
	data->allow_filter_fallback = 1;
149
	data->tree_filter_max_depth = ULONG_MAX;
150
	packet_writer_init(&data->writer, 1);
151
	list_objects_filter_init(&data->filter_options);
152

153
	data->keepalive = 5;
154
	data->advertise_sid = 0;
155
}
156

157
static void upload_pack_data_clear(struct upload_pack_data *data)
158
{
159
	string_list_clear(&data->symref, 1);
160
	strmap_clear(&data->wanted_refs, 1);
161
	strvec_clear(&data->hidden_refs);
162
	object_array_clear(&data->want_obj);
163
	object_array_clear(&data->have_obj);
164
	object_array_clear(&data->shallows);
165
	oidset_clear(&data->deepen_not);
166
	object_array_clear(&data->extra_edge_obj);
167
	list_objects_filter_release(&data->filter_options);
168
	string_list_clear(&data->allowed_filters, 0);
169

170
	free((char *)data->pack_objects_hook);
171
}
172

173
static void reset_timeout(unsigned int timeout)
174
{
175
	alarm(timeout);
176
}
177

178
static void send_client_data(int fd, const char *data, ssize_t sz,
179
			     int use_sideband)
180
{
181
	if (use_sideband) {
182
		send_sideband(1, fd, data, sz, use_sideband);
183
		return;
184
	}
185
	if (fd == 3)
186
		/* emergency quit */
187
		fd = 2;
188
	if (fd == 2) {
189
		/* XXX: are we happy to lose stuff here? */
190
		xwrite(fd, data, sz);
191
		return;
192
	}
193
	write_or_die(fd, data, sz);
194
}
195

196
static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
197
{
198
	FILE *fp = cb_data;
199
	if (graft->nr_parent == -1)
200
		fprintf(fp, "--shallow %s\n", oid_to_hex(&graft->oid));
201
	return 0;
202
}
203

204
struct output_state {
205
	/*
206
	 * We do writes no bigger than LARGE_PACKET_DATA_MAX - 1, because with
207
	 * sideband-64k the band designator takes up 1 byte of space. Because
208
	 * relay_pack_data keeps the last byte to itself, we make the buffer 1
209
	 * byte bigger than the intended maximum write size.
210
	 */
211
	char buffer[(LARGE_PACKET_DATA_MAX - 1) + 1];
212
	int used;
213
	unsigned packfile_uris_started : 1;
214
	unsigned packfile_started : 1;
215
};
216

217
static int relay_pack_data(int pack_objects_out, struct output_state *os,
218
			   int use_sideband, int write_packfile_line)
219
{
220
	/*
221
	 * We keep the last byte to ourselves
222
	 * in case we detect broken rev-list, so that we
223
	 * can leave the stream corrupted.  This is
224
	 * unfortunate -- unpack-objects would happily
225
	 * accept a valid packdata with trailing garbage,
226
	 * so appending garbage after we pass all the
227
	 * pack data is not good enough to signal
228
	 * breakage to downstream.
229
	 */
230
	ssize_t readsz;
231

232
	readsz = xread(pack_objects_out, os->buffer + os->used,
233
		       sizeof(os->buffer) - os->used);
234
	if (readsz < 0) {
235
		return readsz;
236
	}
237
	os->used += readsz;
238

239
	while (!os->packfile_started) {
240
		char *p;
241
		if (os->used >= 4 && !memcmp(os->buffer, "PACK", 4)) {
242
			os->packfile_started = 1;
243
			if (write_packfile_line) {
244
				if (os->packfile_uris_started)
245
					packet_delim(1);
246
				packet_write_fmt(1, "\1packfile\n");
247
			}
248
			break;
249
		}
250
		if ((p = memchr(os->buffer, '\n', os->used))) {
251
			if (!os->packfile_uris_started) {
252
				os->packfile_uris_started = 1;
253
				if (!write_packfile_line)
254
					BUG("packfile_uris requires sideband-all");
255
				packet_write_fmt(1, "\1packfile-uris\n");
256
			}
257
			*p = '\0';
258
			packet_write_fmt(1, "\1%s\n", os->buffer);
259

260
			os->used -= p - os->buffer + 1;
261
			memmove(os->buffer, p + 1, os->used);
262
		} else {
263
			/*
264
			 * Incomplete line.
265
			 */
266
			return readsz;
267
		}
268
	}
269

270
	if (os->used > 1) {
271
		send_client_data(1, os->buffer, os->used - 1, use_sideband);
272
		os->buffer[0] = os->buffer[os->used - 1];
273
		os->used = 1;
274
	} else {
275
		send_client_data(1, os->buffer, os->used, use_sideband);
276
		os->used = 0;
277
	}
278

279
	return readsz;
280
}
281

282
static void create_pack_file(struct upload_pack_data *pack_data,
283
			     const struct string_list *uri_protocols)
284
{
285
	struct child_process pack_objects = CHILD_PROCESS_INIT;
286
	struct output_state *output_state = xcalloc(1, sizeof(struct output_state));
287
	char progress[128];
288
	char abort_msg[] = "aborting due to possible repository "
289
		"corruption on the remote side.";
290
	ssize_t sz;
291
	int i;
292
	FILE *pipe_fd;
293

294
	if (!pack_data->pack_objects_hook)
295
		pack_objects.git_cmd = 1;
296
	else {
297
		strvec_push(&pack_objects.args, pack_data->pack_objects_hook);
298
		strvec_push(&pack_objects.args, "git");
299
		pack_objects.use_shell = 1;
300
	}
301

302
	if (pack_data->shallow_nr) {
303
		strvec_push(&pack_objects.args, "--shallow-file");
304
		strvec_push(&pack_objects.args, "");
305
	}
306
	strvec_push(&pack_objects.args, "pack-objects");
307
	strvec_push(&pack_objects.args, "--revs");
308
	if (pack_data->use_thin_pack)
309
		strvec_push(&pack_objects.args, "--thin");
310

311
	strvec_push(&pack_objects.args, "--stdout");
312
	if (pack_data->shallow_nr)
313
		strvec_push(&pack_objects.args, "--shallow");
314
	if (!pack_data->no_progress)
315
		strvec_push(&pack_objects.args, "--progress");
316
	if (pack_data->use_ofs_delta)
317
		strvec_push(&pack_objects.args, "--delta-base-offset");
318
	if (pack_data->use_include_tag)
319
		strvec_push(&pack_objects.args, "--include-tag");
320
	if (pack_data->filter_options.choice) {
321
		const char *spec =
322
			expand_list_objects_filter_spec(&pack_data->filter_options);
323
		strvec_pushf(&pack_objects.args, "--filter=%s", spec);
324
	}
325
	if (uri_protocols) {
326
		for (i = 0; i < uri_protocols->nr; i++)
327
			strvec_pushf(&pack_objects.args, "--uri-protocol=%s",
328
					 uri_protocols->items[i].string);
329
	}
330

331
	pack_objects.in = -1;
332
	pack_objects.out = -1;
333
	pack_objects.err = -1;
334
	pack_objects.clean_on_exit = 1;
335

336
	if (start_command(&pack_objects))
337
		die("git upload-pack: unable to fork git-pack-objects");
338

339
	pipe_fd = xfdopen(pack_objects.in, "w");
340

341
	if (pack_data->shallow_nr)
342
		for_each_commit_graft(write_one_shallow, pipe_fd);
343

344
	for (i = 0; i < pack_data->want_obj.nr; i++)
345
		fprintf(pipe_fd, "%s\n",
346
			oid_to_hex(&pack_data->want_obj.objects[i].item->oid));
347
	fprintf(pipe_fd, "--not\n");
348
	for (i = 0; i < pack_data->have_obj.nr; i++)
349
		fprintf(pipe_fd, "%s\n",
350
			oid_to_hex(&pack_data->have_obj.objects[i].item->oid));
351
	for (i = 0; i < pack_data->extra_edge_obj.nr; i++)
352
		fprintf(pipe_fd, "%s\n",
353
			oid_to_hex(&pack_data->extra_edge_obj.objects[i].item->oid));
354
	fprintf(pipe_fd, "\n");
355
	fflush(pipe_fd);
356
	fclose(pipe_fd);
357

358
	/* We read from pack_objects.err to capture stderr output for
359
	 * progress bar, and pack_objects.out to capture the pack data.
360
	 */
361

362
	while (1) {
363
		struct pollfd pfd[2];
364
		int pe, pu, pollsize, polltimeout;
365
		int ret;
366

367
		reset_timeout(pack_data->timeout);
368

369
		pollsize = 0;
370
		pe = pu = -1;
371

372
		if (0 <= pack_objects.out) {
373
			pfd[pollsize].fd = pack_objects.out;
374
			pfd[pollsize].events = POLLIN;
375
			pu = pollsize;
376
			pollsize++;
377
		}
378
		if (0 <= pack_objects.err) {
379
			pfd[pollsize].fd = pack_objects.err;
380
			pfd[pollsize].events = POLLIN;
381
			pe = pollsize;
382
			pollsize++;
383
		}
384

385
		if (!pollsize)
386
			break;
387

388
		polltimeout = pack_data->keepalive < 0
389
			? -1
390
			: 1000 * pack_data->keepalive;
391

392
		ret = poll(pfd, pollsize, polltimeout);
393

394
		if (ret < 0) {
395
			if (errno != EINTR) {
396
				error_errno("poll failed, resuming");
397
				sleep(1);
398
			}
399
			continue;
400
		}
401
		if (0 <= pe && (pfd[pe].revents & (POLLIN|POLLHUP))) {
402
			/* Status ready; we ship that in the side-band
403
			 * or dump to the standard error.
404
			 */
405
			sz = xread(pack_objects.err, progress,
406
				  sizeof(progress));
407
			if (0 < sz)
408
				send_client_data(2, progress, sz,
409
						 pack_data->use_sideband);
410
			else if (sz == 0) {
411
				close(pack_objects.err);
412
				pack_objects.err = -1;
413
			}
414
			else
415
				goto fail;
416
			/* give priority to status messages */
417
			continue;
418
		}
419
		if (0 <= pu && (pfd[pu].revents & (POLLIN|POLLHUP))) {
420
			int result = relay_pack_data(pack_objects.out,
421
						     output_state,
422
						     pack_data->use_sideband,
423
						     !!uri_protocols);
424

425
			if (result == 0) {
426
				close(pack_objects.out);
427
				pack_objects.out = -1;
428
			} else if (result < 0) {
429
				goto fail;
430
			}
431
		}
432

433
		/*
434
		 * We hit the keepalive timeout without saying anything; send
435
		 * an empty message on the data sideband just to let the other
436
		 * side know we're still working on it, but don't have any data
437
		 * yet.
438
		 *
439
		 * If we don't have a sideband channel, there's no room in the
440
		 * protocol to say anything, so those clients are just out of
441
		 * luck.
442
		 */
443
		if (!ret && pack_data->use_sideband) {
444
			static const char buf[] = "0005\1";
445
			write_or_die(1, buf, 5);
446
		}
447
	}
448

449
	if (finish_command(&pack_objects)) {
450
		error("git upload-pack: git-pack-objects died with error.");
451
		goto fail;
452
	}
453

454
	/* flush the data */
455
	if (output_state->used > 0) {
456
		send_client_data(1, output_state->buffer, output_state->used,
457
				 pack_data->use_sideband);
458
		fprintf(stderr, "flushed.\n");
459
	}
460
	free(output_state);
461
	if (pack_data->use_sideband)
462
		packet_flush(1);
463
	return;
464

465
 fail:
466
	free(output_state);
467
	send_client_data(3, abort_msg, strlen(abort_msg),
468
			 pack_data->use_sideband);
469
	die("git upload-pack: %s", abort_msg);
470
}
471

472
static int do_got_oid(struct upload_pack_data *data, const struct object_id *oid)
473
{
474
	int we_knew_they_have = 0;
475
	struct object *o = parse_object_with_flags(the_repository, oid,
476
						   PARSE_OBJECT_SKIP_HASH_CHECK |
477
						   PARSE_OBJECT_DISCARD_TREE);
478

479
	if (!o)
480
		die("oops (%s)", oid_to_hex(oid));
481
	if (o->type == OBJ_COMMIT) {
482
		struct commit_list *parents;
483
		struct commit *commit = (struct commit *)o;
484
		if (o->flags & THEY_HAVE)
485
			we_knew_they_have = 1;
486
		else
487
			o->flags |= THEY_HAVE;
488
		if (!data->oldest_have || (commit->date < data->oldest_have))
489
			data->oldest_have = commit->date;
490
		for (parents = commit->parents;
491
		     parents;
492
		     parents = parents->next)
493
			parents->item->object.flags |= THEY_HAVE;
494
	}
495
	if (!we_knew_they_have) {
496
		add_object_array(o, NULL, &data->have_obj);
497
		return 1;
498
	}
499
	return 0;
500
}
501

502
static int got_oid(struct upload_pack_data *data,
503
		   const char *hex, struct object_id *oid)
504
{
505
	if (get_oid_hex(hex, oid))
506
		die("git upload-pack: expected SHA1 object, got '%s'", hex);
507
	if (!repo_has_object_file_with_flags(the_repository, oid,
508
					     OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT))
509
		return -1;
510
	return do_got_oid(data, oid);
511
}
512

513
static int ok_to_give_up(struct upload_pack_data *data)
514
{
515
	timestamp_t min_generation = GENERATION_NUMBER_ZERO;
516

517
	if (!data->have_obj.nr)
518
		return 0;
519

520
	return can_all_from_reach_with_flag(&data->want_obj, THEY_HAVE,
521
					    COMMON_KNOWN, data->oldest_have,
522
					    min_generation);
523
}
524

525
static int get_common_commits(struct upload_pack_data *data,
526
			      struct packet_reader *reader)
527
{
528
	struct object_id oid;
529
	char last_hex[GIT_MAX_HEXSZ + 1];
530
	int got_common = 0;
531
	int got_other = 0;
532
	int sent_ready = 0;
533

534
	for (;;) {
535
		const char *arg;
536

537
		reset_timeout(data->timeout);
538

539
		if (packet_reader_read(reader) != PACKET_READ_NORMAL) {
540
			if (data->multi_ack == MULTI_ACK_DETAILED
541
			    && got_common
542
			    && !got_other
543
			    && ok_to_give_up(data)) {
544
				sent_ready = 1;
545
				packet_write_fmt(1, "ACK %s ready\n", last_hex);
546
			}
547
			if (data->have_obj.nr == 0 || data->multi_ack)
548
				packet_write_fmt(1, "NAK\n");
549

550
			if (data->no_done && sent_ready) {
551
				packet_write_fmt(1, "ACK %s\n", last_hex);
552
				return 0;
553
			}
554
			if (data->stateless_rpc)
555
				exit(0);
556
			got_common = 0;
557
			got_other = 0;
558
			continue;
559
		}
560
		if (skip_prefix(reader->line, "have ", &arg)) {
561
			switch (got_oid(data, arg, &oid)) {
562
			case -1: /* they have what we do not */
563
				got_other = 1;
564
				if (data->multi_ack
565
				    && ok_to_give_up(data)) {
566
					const char *hex = oid_to_hex(&oid);
567
					if (data->multi_ack == MULTI_ACK_DETAILED) {
568
						sent_ready = 1;
569
						packet_write_fmt(1, "ACK %s ready\n", hex);
570
					} else
571
						packet_write_fmt(1, "ACK %s continue\n", hex);
572
				}
573
				break;
574
			default:
575
				got_common = 1;
576
				oid_to_hex_r(last_hex, &oid);
577
				if (data->multi_ack == MULTI_ACK_DETAILED)
578
					packet_write_fmt(1, "ACK %s common\n", last_hex);
579
				else if (data->multi_ack)
580
					packet_write_fmt(1, "ACK %s continue\n", last_hex);
581
				else if (data->have_obj.nr == 1)
582
					packet_write_fmt(1, "ACK %s\n", last_hex);
583
				break;
584
			}
585
			continue;
586
		}
587
		if (!strcmp(reader->line, "done")) {
588
			if (data->have_obj.nr > 0) {
589
				if (data->multi_ack)
590
					packet_write_fmt(1, "ACK %s\n", last_hex);
591
				return 0;
592
			}
593
			packet_write_fmt(1, "NAK\n");
594
			return -1;
595
		}
596
		die("git upload-pack: expected SHA1 list, got '%s'", reader->line);
597
	}
598
}
599

600
static int allow_hidden_refs(enum allow_uor allow_uor)
601
{
602
	if ((allow_uor & ALLOW_ANY_SHA1) == ALLOW_ANY_SHA1)
603
		return 1;
604
	return !(allow_uor & (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
605
}
606

607
static void for_each_namespaced_ref_1(each_ref_fn fn,
608
				      struct upload_pack_data *data)
609
{
610
	const char **excludes = NULL;
611
	/*
612
	 * If `data->allow_uor` allows fetching hidden refs, we need to
613
	 * mark all references (including hidden ones), to check in
614
	 * `is_our_ref()` below.
615
	 *
616
	 * Otherwise, we only care about whether each reference's object
617
	 * has the OUR_REF bit set or not, so do not need to visit
618
	 * hidden references.
619
	 */
620
	if (allow_hidden_refs(data->allow_uor))
621
		excludes = hidden_refs_to_excludes(&data->hidden_refs);
622

623
	refs_for_each_namespaced_ref(get_main_ref_store(the_repository),
624
				     excludes, fn, data);
625
}
626

627

628
static int is_our_ref(struct object *o, enum allow_uor allow_uor)
629
{
630
	return o->flags & ((allow_hidden_refs(allow_uor) ? 0 : HIDDEN_REF) | OUR_REF);
631
}
632

633
/*
634
 * on successful case, it's up to the caller to close cmd->out
635
 */
636
static int do_reachable_revlist(struct child_process *cmd,
637
				struct object_array *src,
638
				struct object_array *reachable,
639
				enum allow_uor allow_uor)
640
{
641
	struct object *o;
642
	FILE *cmd_in = NULL;
643
	int i;
644

645
	strvec_pushl(&cmd->args, "rev-list", "--stdin", NULL);
646
	cmd->git_cmd = 1;
647
	cmd->no_stderr = 1;
648
	cmd->in = -1;
649
	cmd->out = -1;
650

651
	/*
652
	 * If the next rev-list --stdin encounters an unknown commit,
653
	 * it terminates, which will cause SIGPIPE in the write loop
654
	 * below.
655
	 */
656
	sigchain_push(SIGPIPE, SIG_IGN);
657

658
	if (start_command(cmd))
659
		goto error;
660

661
	cmd_in = xfdopen(cmd->in, "w");
662

663
	for (i = get_max_object_index(); 0 < i; ) {
664
		o = get_indexed_object(--i);
665
		if (!o)
666
			continue;
667
		if (reachable && o->type == OBJ_COMMIT)
668
			o->flags &= ~TMP_MARK;
669
		if (!is_our_ref(o, allow_uor))
670
			continue;
671
		if (fprintf(cmd_in, "^%s\n", oid_to_hex(&o->oid)) < 0)
672
			goto error;
673
	}
674
	for (i = 0; i < src->nr; i++) {
675
		o = src->objects[i].item;
676
		if (is_our_ref(o, allow_uor)) {
677
			if (reachable)
678
				add_object_array(o, NULL, reachable);
679
			continue;
680
		}
681
		if (reachable && o->type == OBJ_COMMIT)
682
			o->flags |= TMP_MARK;
683
		if (fprintf(cmd_in, "%s\n", oid_to_hex(&o->oid)) < 0)
684
			goto error;
685
	}
686
	if (ferror(cmd_in) || fflush(cmd_in))
687
		goto error;
688
	fclose(cmd_in);
689
	cmd->in = -1;
690
	sigchain_pop(SIGPIPE);
691

692
	return 0;
693

694
error:
695
	sigchain_pop(SIGPIPE);
696

697
	if (cmd_in)
698
		fclose(cmd_in);
699
	if (cmd->out >= 0)
700
		close(cmd->out);
701
	return -1;
702
}
703

704
static int get_reachable_list(struct upload_pack_data *data,
705
			      struct object_array *reachable)
706
{
707
	struct child_process cmd = CHILD_PROCESS_INIT;
708
	int i;
709
	struct object *o;
710
	char namebuf[GIT_MAX_HEXSZ + 2]; /* ^ + hash + LF */
711
	const unsigned hexsz = the_hash_algo->hexsz;
712

713
	if (do_reachable_revlist(&cmd, &data->shallows, reachable,
714
				 data->allow_uor) < 0)
715
		return -1;
716

717
	while ((i = read_in_full(cmd.out, namebuf, hexsz + 1)) == hexsz + 1) {
718
		struct object_id oid;
719
		const char *p;
720

721
		if (parse_oid_hex(namebuf, &oid, &p) || *p != '\n')
722
			break;
723

724
		o = lookup_object(the_repository, &oid);
725
		if (o && o->type == OBJ_COMMIT) {
726
			o->flags &= ~TMP_MARK;
727
		}
728
	}
729
	for (i = get_max_object_index(); 0 < i; i--) {
730
		o = get_indexed_object(i - 1);
731
		if (o && o->type == OBJ_COMMIT &&
732
		    (o->flags & TMP_MARK)) {
733
			add_object_array(o, NULL, reachable);
734
				o->flags &= ~TMP_MARK;
735
		}
736
	}
737
	close(cmd.out);
738

739
	if (finish_command(&cmd))
740
		return -1;
741

742
	return 0;
743
}
744

745
static int has_unreachable(struct object_array *src, enum allow_uor allow_uor)
746
{
747
	struct child_process cmd = CHILD_PROCESS_INIT;
748
	char buf[1];
749
	int i;
750

751
	if (do_reachable_revlist(&cmd, src, NULL, allow_uor) < 0)
752
		return 1;
753

754
	/*
755
	 * The commits out of the rev-list are not ancestors of
756
	 * our ref.
757
	 */
758
	i = read_in_full(cmd.out, buf, 1);
759
	if (i)
760
		goto error;
761
	close(cmd.out);
762
	cmd.out = -1;
763

764
	/*
765
	 * rev-list may have died by encountering a bad commit
766
	 * in the history, in which case we do want to bail out
767
	 * even when it showed no commit.
768
	 */
769
	if (finish_command(&cmd))
770
		goto error;
771

772
	/* All the non-tip ones are ancestors of what we advertised */
773
	return 0;
774

775
error:
776
	if (cmd.out >= 0)
777
		close(cmd.out);
778
	return 1;
779
}
780

781
static void check_non_tip(struct upload_pack_data *data)
782
{
783
	int i;
784

785
	/*
786
	 * In the normal in-process case without
787
	 * uploadpack.allowReachableSHA1InWant,
788
	 * non-tip requests can never happen.
789
	 */
790
	if (!data->stateless_rpc && !(data->allow_uor & ALLOW_REACHABLE_SHA1))
791
		goto error;
792
	if (!has_unreachable(&data->want_obj, data->allow_uor))
793
		/* All the non-tip ones are ancestors of what we advertised */
794
		return;
795

796
error:
797
	/* Pick one of them (we know there at least is one) */
798
	for (i = 0; i < data->want_obj.nr; i++) {
799
		struct object *o = data->want_obj.objects[i].item;
800
		if (!is_our_ref(o, data->allow_uor)) {
801
			error("git upload-pack: not our ref %s",
802
			      oid_to_hex(&o->oid));
803
			packet_writer_error(&data->writer,
804
					    "upload-pack: not our ref %s",
805
					    oid_to_hex(&o->oid));
806
			exit(128);
807
		}
808
	}
809
}
810

811
static void send_shallow(struct upload_pack_data *data,
812
			 struct commit_list *result)
813
{
814
	while (result) {
815
		struct object *object = &result->item->object;
816
		if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
817
			packet_writer_write(&data->writer, "shallow %s",
818
					    oid_to_hex(&object->oid));
819
			register_shallow(the_repository, &object->oid);
820
			data->shallow_nr++;
821
		}
822
		result = result->next;
823
	}
824
}
825

826
static void send_unshallow(struct upload_pack_data *data)
827
{
828
	int i;
829

830
	for (i = 0; i < data->shallows.nr; i++) {
831
		struct object *object = data->shallows.objects[i].item;
832
		if (object->flags & NOT_SHALLOW) {
833
			struct commit_list *parents;
834
			packet_writer_write(&data->writer, "unshallow %s",
835
					    oid_to_hex(&object->oid));
836
			object->flags &= ~CLIENT_SHALLOW;
837
			/*
838
			 * We want to _register_ "object" as shallow, but we
839
			 * also need to traverse object's parents to deepen a
840
			 * shallow clone. Unregister it for now so we can
841
			 * parse and add the parents to the want list, then
842
			 * re-register it.
843
			 */
844
			unregister_shallow(&object->oid);
845
			object->parsed = 0;
846
			parse_commit_or_die((struct commit *)object);
847
			parents = ((struct commit *)object)->parents;
848
			while (parents) {
849
				add_object_array(&parents->item->object,
850
						 NULL, &data->want_obj);
851
				parents = parents->next;
852
			}
853
			add_object_array(object, NULL, &data->extra_edge_obj);
854
		}
855
		/* make sure commit traversal conforms to client */
856
		register_shallow(the_repository, &object->oid);
857
	}
858
}
859

860
static int check_ref(const char *refname_full, const char *referent UNUSED, const struct object_id *oid,
861
		     int flag, void *cb_data);
862
static void deepen(struct upload_pack_data *data, int depth)
863
{
864
	if (depth == INFINITE_DEPTH && !is_repository_shallow(the_repository)) {
865
		int i;
866

867
		for (i = 0; i < data->shallows.nr; i++) {
868
			struct object *object = data->shallows.objects[i].item;
869
			object->flags |= NOT_SHALLOW;
870
		}
871
	} else if (data->deepen_relative) {
872
		struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
873
		struct commit_list *result;
874

875
		/*
876
		 * Checking for reachable shallows requires that our refs be
877
		 * marked with OUR_REF.
878
		 */
879
		refs_head_ref_namespaced(get_main_ref_store(the_repository),
880
					 check_ref, data);
881
		for_each_namespaced_ref_1(check_ref, data);
882

883
		get_reachable_list(data, &reachable_shallows);
884
		result = get_shallow_commits(&reachable_shallows,
885
					     depth + 1,
886
					     SHALLOW, NOT_SHALLOW);
887
		send_shallow(data, result);
888
		free_commit_list(result);
889
		object_array_clear(&reachable_shallows);
890
	} else {
891
		struct commit_list *result;
892

893
		result = get_shallow_commits(&data->want_obj, depth,
894
					     SHALLOW, NOT_SHALLOW);
895
		send_shallow(data, result);
896
		free_commit_list(result);
897
	}
898

899
	send_unshallow(data);
900
}
901

902
static void deepen_by_rev_list(struct upload_pack_data *data,
903
			       int ac,
904
			       const char **av)
905
{
906
	struct commit_list *result;
907

908
	disable_commit_graph(the_repository);
909
	result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW);
910
	send_shallow(data, result);
911
	free_commit_list(result);
912
	send_unshallow(data);
913
}
914

915
/* Returns 1 if a shallow list is sent or 0 otherwise */
916
static int send_shallow_list(struct upload_pack_data *data)
917
{
918
	int ret = 0;
919

920
	if (data->depth > 0 && data->deepen_rev_list)
921
		die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
922
	if (data->depth > 0) {
923
		deepen(data, data->depth);
924
		ret = 1;
925
	} else if (data->deepen_rev_list) {
926
		struct strvec av = STRVEC_INIT;
927
		int i;
928

929
		strvec_push(&av, "rev-list");
930
		if (data->deepen_since)
931
			strvec_pushf(&av, "--max-age=%"PRItime, data->deepen_since);
932
		if (oidset_size(&data->deepen_not)) {
933
			const struct object_id *oid;
934
			struct oidset_iter iter;
935
			strvec_push(&av, "--not");
936
			oidset_iter_init(&data->deepen_not, &iter);
937
			while ((oid = oidset_iter_next(&iter)))
938
				strvec_push(&av, oid_to_hex(oid));
939
			strvec_push(&av, "--not");
940
		}
941
		for (i = 0; i < data->want_obj.nr; i++) {
942
			struct object *o = data->want_obj.objects[i].item;
943
			strvec_push(&av, oid_to_hex(&o->oid));
944
		}
945
		deepen_by_rev_list(data, av.nr, av.v);
946
		strvec_clear(&av);
947
		ret = 1;
948
	} else {
949
		if (data->shallows.nr > 0) {
950
			int i;
951
			for (i = 0; i < data->shallows.nr; i++)
952
				register_shallow(the_repository,
953
						 &data->shallows.objects[i].item->oid);
954
		}
955
	}
956

957
	data->shallow_nr += data->shallows.nr;
958
	return ret;
959
}
960

961
static int process_shallow(const char *line, struct object_array *shallows)
962
{
963
	const char *arg;
964
	if (skip_prefix(line, "shallow ", &arg)) {
965
		struct object_id oid;
966
		struct object *object;
967
		if (get_oid_hex(arg, &oid))
968
			die("invalid shallow line: %s", line);
969
		object = parse_object(the_repository, &oid);
970
		if (!object)
971
			return 1;
972
		if (object->type != OBJ_COMMIT)
973
			die("invalid shallow object %s", oid_to_hex(&oid));
974
		if (!(object->flags & CLIENT_SHALLOW)) {
975
			object->flags |= CLIENT_SHALLOW;
976
			add_object_array(object, NULL, shallows);
977
		}
978
		return 1;
979
	}
980

981
	return 0;
982
}
983

984
static int process_deepen(const char *line, int *depth)
985
{
986
	const char *arg;
987
	if (skip_prefix(line, "deepen ", &arg)) {
988
		char *end = NULL;
989
		*depth = (int)strtol(arg, &end, 0);
990
		if (!end || *end || *depth <= 0)
991
			die("Invalid deepen: %s", line);
992
		return 1;
993
	}
994

995
	return 0;
996
}
997

998
static int process_deepen_since(const char *line, timestamp_t *deepen_since, int *deepen_rev_list)
999
{
1000
	const char *arg;
1001
	if (skip_prefix(line, "deepen-since ", &arg)) {
1002
		char *end = NULL;
1003
		*deepen_since = parse_timestamp(arg, &end, 0);
1004
		if (!end || *end || !deepen_since ||
1005
		    /* revisions.c's max_age -1 is special */
1006
		    *deepen_since == -1)
1007
			die("Invalid deepen-since: %s", line);
1008
		*deepen_rev_list = 1;
1009
		return 1;
1010
	}
1011
	return 0;
1012
}
1013

1014
static int process_deepen_not(const char *line, struct oidset *deepen_not, int *deepen_rev_list)
1015
{
1016
	const char *arg;
1017
	if (skip_prefix(line, "deepen-not ", &arg)) {
1018
		char *ref = NULL;
1019
		struct object_id oid;
1020
		if (expand_ref(the_repository, arg, strlen(arg), &oid, &ref) != 1)
1021
			die("git upload-pack: ambiguous deepen-not: %s", line);
1022
		oidset_insert(deepen_not, &oid);
1023
		free(ref);
1024
		*deepen_rev_list = 1;
1025
		return 1;
1026
	}
1027
	return 0;
1028
}
1029

1030
NORETURN __attribute__((format(printf,2,3)))
1031
static void send_err_and_die(struct upload_pack_data *data,
1032
			     const char *fmt, ...)
1033
{
1034
	struct strbuf buf = STRBUF_INIT;
1035
	va_list ap;
1036

1037
	va_start(ap, fmt);
1038
	strbuf_vaddf(&buf, fmt, ap);
1039
	va_end(ap);
1040

1041
	packet_writer_error(&data->writer, "%s", buf.buf);
1042
	die("%s", buf.buf);
1043
}
1044

1045
static void check_one_filter(struct upload_pack_data *data,
1046
			     struct list_objects_filter_options *opts)
1047
{
1048
	const char *key = list_object_filter_config_name(opts->choice);
1049
	struct string_list_item *item = string_list_lookup(&data->allowed_filters,
1050
							   key);
1051
	int allowed;
1052

1053
	if (item)
1054
		allowed = (intptr_t)item->util;
1055
	else
1056
		allowed = data->allow_filter_fallback;
1057

1058
	if (!allowed)
1059
		send_err_and_die(data, "filter '%s' not supported", key);
1060

1061
	if (opts->choice == LOFC_TREE_DEPTH &&
1062
	    opts->tree_exclude_depth > data->tree_filter_max_depth)
1063
		send_err_and_die(data,
1064
				 "tree filter allows max depth %lu, but got %lu",
1065
				 data->tree_filter_max_depth,
1066
				 opts->tree_exclude_depth);
1067
}
1068

1069
static void check_filter_recurse(struct upload_pack_data *data,
1070
				 struct list_objects_filter_options *opts)
1071
{
1072
	size_t i;
1073

1074
	check_one_filter(data, opts);
1075
	if (opts->choice != LOFC_COMBINE)
1076
		return;
1077

1078
	for (i = 0; i < opts->sub_nr; i++)
1079
		check_filter_recurse(data, &opts->sub[i]);
1080
}
1081

1082
static void die_if_using_banned_filter(struct upload_pack_data *data)
1083
{
1084
	check_filter_recurse(data, &data->filter_options);
1085
}
1086

1087
static void receive_needs(struct upload_pack_data *data,
1088
			  struct packet_reader *reader)
1089
{
1090
	int has_non_tip = 0;
1091

1092
	data->shallow_nr = 0;
1093
	for (;;) {
1094
		struct object *o;
1095
		const char *features;
1096
		struct object_id oid_buf;
1097
		const char *arg;
1098
		size_t feature_len;
1099

1100
		reset_timeout(data->timeout);
1101
		if (packet_reader_read(reader) != PACKET_READ_NORMAL)
1102
			break;
1103

1104
		if (process_shallow(reader->line, &data->shallows))
1105
			continue;
1106
		if (process_deepen(reader->line, &data->depth))
1107
			continue;
1108
		if (process_deepen_since(reader->line, &data->deepen_since, &data->deepen_rev_list))
1109
			continue;
1110
		if (process_deepen_not(reader->line, &data->deepen_not, &data->deepen_rev_list))
1111
			continue;
1112

1113
		if (skip_prefix(reader->line, "filter ", &arg)) {
1114
			if (!data->filter_capability_requested)
1115
				die("git upload-pack: filtering capability not negotiated");
1116
			list_objects_filter_die_if_populated(&data->filter_options);
1117
			parse_list_objects_filter(&data->filter_options, arg);
1118
			die_if_using_banned_filter(data);
1119
			continue;
1120
		}
1121

1122
		if (!skip_prefix(reader->line, "want ", &arg) ||
1123
		    parse_oid_hex(arg, &oid_buf, &features))
1124
			die("git upload-pack: protocol error, "
1125
			    "expected to get object ID, not '%s'", reader->line);
1126

1127
		if (parse_feature_request(features, "deepen-relative"))
1128
			data->deepen_relative = 1;
1129
		if (parse_feature_request(features, "multi_ack_detailed"))
1130
			data->multi_ack = MULTI_ACK_DETAILED;
1131
		else if (parse_feature_request(features, "multi_ack"))
1132
			data->multi_ack = MULTI_ACK;
1133
		if (parse_feature_request(features, "no-done"))
1134
			data->no_done = 1;
1135
		if (parse_feature_request(features, "thin-pack"))
1136
			data->use_thin_pack = 1;
1137
		if (parse_feature_request(features, "ofs-delta"))
1138
			data->use_ofs_delta = 1;
1139
		if (parse_feature_request(features, "side-band-64k"))
1140
			data->use_sideband = LARGE_PACKET_MAX;
1141
		else if (parse_feature_request(features, "side-band"))
1142
			data->use_sideband = DEFAULT_PACKET_MAX;
1143
		if (parse_feature_request(features, "no-progress"))
1144
			data->no_progress = 1;
1145
		if (parse_feature_request(features, "include-tag"))
1146
			data->use_include_tag = 1;
1147
		if (data->allow_filter &&
1148
		    parse_feature_request(features, "filter"))
1149
			data->filter_capability_requested = 1;
1150

1151
		arg = parse_feature_value(features, "session-id", &feature_len, NULL);
1152
		if (arg) {
1153
			char *client_sid = xstrndup(arg, feature_len);
1154
			trace2_data_string("transfer", NULL, "client-sid", client_sid);
1155
			free(client_sid);
1156
		}
1157

1158
		o = parse_object_with_flags(the_repository, &oid_buf,
1159
					    PARSE_OBJECT_SKIP_HASH_CHECK |
1160
					    PARSE_OBJECT_DISCARD_TREE);
1161
		if (!o) {
1162
			packet_writer_error(&data->writer,
1163
					    "upload-pack: not our ref %s",
1164
					    oid_to_hex(&oid_buf));
1165
			die("git upload-pack: not our ref %s",
1166
			    oid_to_hex(&oid_buf));
1167
		}
1168
		if (!(o->flags & WANTED)) {
1169
			o->flags |= WANTED;
1170
			if (!((data->allow_uor & ALLOW_ANY_SHA1) == ALLOW_ANY_SHA1
1171
			      || is_our_ref(o, data->allow_uor)))
1172
				has_non_tip = 1;
1173
			add_object_array(o, NULL, &data->want_obj);
1174
		}
1175
	}
1176

1177
	/*
1178
	 * We have sent all our refs already, and the other end
1179
	 * should have chosen out of them. When we are operating
1180
	 * in the stateless RPC mode, however, their choice may
1181
	 * have been based on the set of older refs advertised
1182
	 * by another process that handled the initial request.
1183
	 */
1184
	if (has_non_tip)
1185
		check_non_tip(data);
1186

1187
	if (!data->use_sideband && data->daemon_mode)
1188
		data->no_progress = 1;
1189

1190
	if (data->depth == 0 && !data->deepen_rev_list && data->shallows.nr == 0)
1191
		return;
1192

1193
	if (send_shallow_list(data))
1194
		packet_flush(1);
1195
}
1196

1197
/* return non-zero if the ref is hidden, otherwise 0 */
1198
static int mark_our_ref(const char *refname, const char *refname_full,
1199
			const struct object_id *oid, const struct strvec *hidden_refs)
1200
{
1201
	struct object *o = lookup_unknown_object(the_repository, oid);
1202

1203
	if (ref_is_hidden(refname, refname_full, hidden_refs)) {
1204
		o->flags |= HIDDEN_REF;
1205
		return 1;
1206
	}
1207
	o->flags |= OUR_REF;
1208
	return 0;
1209
}
1210

1211
static int check_ref(const char *refname_full, const char *referent UNUSED,const struct object_id *oid,
1212
		     int flag UNUSED, void *cb_data)
1213
{
1214
	const char *refname = strip_namespace(refname_full);
1215
	struct upload_pack_data *data = cb_data;
1216

1217
	mark_our_ref(refname, refname_full, oid, &data->hidden_refs);
1218
	return 0;
1219
}
1220

1221
static void format_symref_info(struct strbuf *buf, struct string_list *symref)
1222
{
1223
	struct string_list_item *item;
1224

1225
	if (!symref->nr)
1226
		return;
1227
	for_each_string_list_item(item, symref)
1228
		strbuf_addf(buf, " symref=%s:%s", item->string, (char *)item->util);
1229
}
1230

1231
static void format_session_id(struct strbuf *buf, struct upload_pack_data *d) {
1232
	if (d->advertise_sid)
1233
		strbuf_addf(buf, " session-id=%s", trace2_session_id());
1234
}
1235

1236
static void write_v0_ref(struct upload_pack_data *data,
1237
			const char *refname, const char *refname_nons,
1238
			const struct object_id *oid)
1239
{
1240
	static const char *capabilities = "multi_ack thin-pack side-band"
1241
		" side-band-64k ofs-delta shallow deepen-since deepen-not"
1242
		" deepen-relative no-progress include-tag multi_ack_detailed";
1243
	struct object_id peeled;
1244

1245
	if (mark_our_ref(refname_nons, refname, oid, &data->hidden_refs))
1246
		return;
1247

1248
	if (capabilities) {
1249
		struct strbuf symref_info = STRBUF_INIT;
1250
		struct strbuf session_id = STRBUF_INIT;
1251

1252
		format_symref_info(&symref_info, &data->symref);
1253
		format_session_id(&session_id, data);
1254
		packet_fwrite_fmt(stdout, "%s %s%c%s%s%s%s%s%s%s object-format=%s agent=%s\n",
1255
			     oid_to_hex(oid), refname_nons,
1256
			     0, capabilities,
1257
			     (data->allow_uor & ALLOW_TIP_SHA1) ?
1258
				     " allow-tip-sha1-in-want" : "",
1259
			     (data->allow_uor & ALLOW_REACHABLE_SHA1) ?
1260
				     " allow-reachable-sha1-in-want" : "",
1261
			     data->no_done ? " no-done" : "",
1262
			     symref_info.buf,
1263
			     data->allow_filter ? " filter" : "",
1264
			     session_id.buf,
1265
			     the_hash_algo->name,
1266
			     git_user_agent_sanitized());
1267
		strbuf_release(&symref_info);
1268
		strbuf_release(&session_id);
1269
		data->sent_capabilities = 1;
1270
	} else {
1271
		packet_fwrite_fmt(stdout, "%s %s\n", oid_to_hex(oid), refname_nons);
1272
	}
1273
	capabilities = NULL;
1274
	if (!peel_iterated_oid(the_repository, oid, &peeled))
1275
		packet_fwrite_fmt(stdout, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons);
1276
	return;
1277
}
1278

1279
static int send_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid,
1280
		    int flag UNUSED, void *cb_data)
1281
{
1282
	write_v0_ref(cb_data, refname, strip_namespace(refname), oid);
1283
	return 0;
1284
}
1285

1286
static int find_symref(const char *refname, const char *referent UNUSED,
1287
		       const struct object_id *oid UNUSED,
1288
		       int flag, void *cb_data)
1289
{
1290
	const char *symref_target;
1291
	struct string_list_item *item;
1292

1293
	if ((flag & REF_ISSYMREF) == 0)
1294
		return 0;
1295
	symref_target = refs_resolve_ref_unsafe(get_main_ref_store(the_repository),
1296
						refname, 0, NULL, &flag);
1297
	if (!symref_target || (flag & REF_ISSYMREF) == 0)
1298
		die("'%s' is a symref but it is not?", refname);
1299
	item = string_list_append(cb_data, strip_namespace(refname));
1300
	item->util = xstrdup(strip_namespace(symref_target));
1301
	return 0;
1302
}
1303

1304
static int parse_object_filter_config(const char *var, const char *value,
1305
				      const struct key_value_info *kvi,
1306
				      struct upload_pack_data *data)
1307
{
1308
	struct strbuf buf = STRBUF_INIT;
1309
	const char *sub, *key;
1310
	size_t sub_len;
1311

1312
	if (parse_config_key(var, "uploadpackfilter", &sub, &sub_len, &key))
1313
		return 0;
1314

1315
	if (!sub) {
1316
		if (!strcmp(key, "allow"))
1317
			data->allow_filter_fallback = git_config_bool(var, value);
1318
		return 0;
1319
	}
1320

1321
	strbuf_add(&buf, sub, sub_len);
1322

1323
	if (!strcmp(key, "allow"))
1324
		string_list_insert(&data->allowed_filters, buf.buf)->util =
1325
			(void *)(intptr_t)git_config_bool(var, value);
1326
	else if (!strcmp(buf.buf, "tree") && !strcmp(key, "maxdepth")) {
1327
		if (!value) {
1328
			strbuf_release(&buf);
1329
			return config_error_nonbool(var);
1330
		}
1331
		string_list_insert(&data->allowed_filters, buf.buf)->util =
1332
			(void *)(intptr_t)1;
1333
		data->tree_filter_max_depth = git_config_ulong(var, value,
1334
							       kvi);
1335
	}
1336

1337
	strbuf_release(&buf);
1338
	return 0;
1339
}
1340

1341
static int upload_pack_config(const char *var, const char *value,
1342
			      const struct config_context *ctx,
1343
			      void *cb_data)
1344
{
1345
	struct upload_pack_data *data = cb_data;
1346

1347
	if (!strcmp("uploadpack.allowtipsha1inwant", var)) {
1348
		if (git_config_bool(var, value))
1349
			data->allow_uor |= ALLOW_TIP_SHA1;
1350
		else
1351
			data->allow_uor &= ~ALLOW_TIP_SHA1;
1352
	} else if (!strcmp("uploadpack.allowreachablesha1inwant", var)) {
1353
		if (git_config_bool(var, value))
1354
			data->allow_uor |= ALLOW_REACHABLE_SHA1;
1355
		else
1356
			data->allow_uor &= ~ALLOW_REACHABLE_SHA1;
1357
	} else if (!strcmp("uploadpack.allowanysha1inwant", var)) {
1358
		if (git_config_bool(var, value))
1359
			data->allow_uor |= ALLOW_ANY_SHA1;
1360
		else
1361
			data->allow_uor &= ~ALLOW_ANY_SHA1;
1362
	} else if (!strcmp("uploadpack.keepalive", var)) {
1363
		data->keepalive = git_config_int(var, value, ctx->kvi);
1364
		if (!data->keepalive)
1365
			data->keepalive = -1;
1366
	} else if (!strcmp("uploadpack.allowfilter", var)) {
1367
		data->allow_filter = git_config_bool(var, value);
1368
	} else if (!strcmp("uploadpack.allowrefinwant", var)) {
1369
		data->allow_ref_in_want = git_config_bool(var, value);
1370
	} else if (!strcmp("uploadpack.allowsidebandall", var)) {
1371
		data->allow_sideband_all = git_config_bool(var, value);
1372
	} else if (!strcmp("uploadpack.blobpackfileuri", var)) {
1373
		if (value)
1374
			data->allow_packfile_uris = 1;
1375
	} else if (!strcmp("core.precomposeunicode", var)) {
1376
		precomposed_unicode = git_config_bool(var, value);
1377
	} else if (!strcmp("transfer.advertisesid", var)) {
1378
		data->advertise_sid = git_config_bool(var, value);
1379
	}
1380

1381
	if (parse_object_filter_config(var, value, ctx->kvi, data) < 0)
1382
		return -1;
1383

1384
	return parse_hide_refs_config(var, value, "uploadpack", &data->hidden_refs);
1385
}
1386

1387
static int upload_pack_protected_config(const char *var, const char *value,
1388
					const struct config_context *ctx UNUSED,
1389
					void *cb_data)
1390
{
1391
	struct upload_pack_data *data = cb_data;
1392

1393
	if (!strcmp("uploadpack.packobjectshook", var))
1394
		return git_config_string(&data->pack_objects_hook, var, value);
1395
	return 0;
1396
}
1397

1398
static void get_upload_pack_config(struct repository *r,
1399
				   struct upload_pack_data *data)
1400
{
1401
	repo_config(r, upload_pack_config, data);
1402
	git_protected_config(upload_pack_protected_config, data);
1403

1404
	data->allow_sideband_all |= git_env_bool("GIT_TEST_SIDEBAND_ALL", 0);
1405
}
1406

1407
void upload_pack(const int advertise_refs, const int stateless_rpc,
1408
		 const int timeout)
1409
{
1410
	struct packet_reader reader;
1411
	struct upload_pack_data data;
1412

1413
	upload_pack_data_init(&data);
1414
	get_upload_pack_config(the_repository, &data);
1415

1416
	data.stateless_rpc = stateless_rpc;
1417
	data.timeout = timeout;
1418
	if (data.timeout)
1419
		data.daemon_mode = 1;
1420

1421
	refs_head_ref_namespaced(get_main_ref_store(the_repository),
1422
				 find_symref, &data.symref);
1423

1424
	if (advertise_refs || !data.stateless_rpc) {
1425
		reset_timeout(data.timeout);
1426
		if (advertise_refs)
1427
			data.no_done = 1;
1428
		refs_head_ref_namespaced(get_main_ref_store(the_repository),
1429
					 send_ref, &data);
1430
		for_each_namespaced_ref_1(send_ref, &data);
1431
		if (!data.sent_capabilities) {
1432
			const char *refname = "capabilities^{}";
1433
			write_v0_ref(&data, refname, refname, null_oid());
1434
		}
1435
		/*
1436
		 * fflush stdout before calling advertise_shallow_grafts because send_ref
1437
		 * uses stdio.
1438
		 */
1439
		fflush_or_die(stdout);
1440
		advertise_shallow_grafts(1);
1441
		packet_flush(1);
1442
	} else {
1443
		refs_head_ref_namespaced(get_main_ref_store(the_repository),
1444
					 check_ref, &data);
1445
		for_each_namespaced_ref_1(check_ref, &data);
1446
	}
1447

1448
	if (!advertise_refs) {
1449
		packet_reader_init(&reader, 0, NULL, 0,
1450
				   PACKET_READ_CHOMP_NEWLINE |
1451
				   PACKET_READ_DIE_ON_ERR_PACKET);
1452

1453
		receive_needs(&data, &reader);
1454

1455
		/*
1456
		 * An EOF at this exact point in negotiation should be
1457
		 * acceptable from stateless clients as they will consume the
1458
		 * shallow list before doing subsequent rpc with haves/etc.
1459
		 */
1460
		if (data.stateless_rpc)
1461
			reader.options |= PACKET_READ_GENTLE_ON_EOF;
1462

1463
		if (data.want_obj.nr &&
1464
		    packet_reader_peek(&reader) != PACKET_READ_EOF) {
1465
			reader.options &= ~PACKET_READ_GENTLE_ON_EOF;
1466
			get_common_commits(&data, &reader);
1467
			create_pack_file(&data, NULL);
1468
		}
1469
	}
1470

1471
	upload_pack_data_clear(&data);
1472
}
1473

1474
static int parse_want(struct packet_writer *writer, const char *line,
1475
		      struct object_array *want_obj)
1476
{
1477
	const char *arg;
1478
	if (skip_prefix(line, "want ", &arg)) {
1479
		struct object_id oid;
1480
		struct object *o;
1481

1482
		if (get_oid_hex(arg, &oid))
1483
			die("git upload-pack: protocol error, "
1484
			    "expected to get oid, not '%s'", line);
1485

1486
		o = parse_object_with_flags(the_repository, &oid,
1487
					    PARSE_OBJECT_SKIP_HASH_CHECK |
1488
					    PARSE_OBJECT_DISCARD_TREE);
1489

1490
		if (!o) {
1491
			packet_writer_error(writer,
1492
					    "upload-pack: not our ref %s",
1493
					    oid_to_hex(&oid));
1494
			die("git upload-pack: not our ref %s",
1495
			    oid_to_hex(&oid));
1496
		}
1497

1498
		if (!(o->flags & WANTED)) {
1499
			o->flags |= WANTED;
1500
			add_object_array(o, NULL, want_obj);
1501
		}
1502

1503
		return 1;
1504
	}
1505

1506
	return 0;
1507
}
1508

1509
static int parse_want_ref(struct packet_writer *writer, const char *line,
1510
			  struct strmap *wanted_refs,
1511
			  struct strvec *hidden_refs,
1512
			  struct object_array *want_obj)
1513
{
1514
	const char *refname_nons;
1515
	if (skip_prefix(line, "want-ref ", &refname_nons)) {
1516
		struct object_id oid;
1517
		struct object *o = NULL;
1518
		struct strbuf refname = STRBUF_INIT;
1519

1520
		strbuf_addf(&refname, "%s%s", get_git_namespace(), refname_nons);
1521
		if (ref_is_hidden(refname_nons, refname.buf, hidden_refs) ||
1522
		    refs_read_ref(get_main_ref_store(the_repository), refname.buf, &oid)) {
1523
			packet_writer_error(writer, "unknown ref %s", refname_nons);
1524
			die("unknown ref %s", refname_nons);
1525
		}
1526
		strbuf_release(&refname);
1527

1528
		if (strmap_put(wanted_refs, refname_nons, oiddup(&oid))) {
1529
			packet_writer_error(writer, "duplicate want-ref %s",
1530
					    refname_nons);
1531
			die("duplicate want-ref %s", refname_nons);
1532
		}
1533

1534
		if (!starts_with(refname_nons, "refs/tags/")) {
1535
			struct commit *commit = lookup_commit_in_graph(the_repository, &oid);
1536
			if (commit)
1537
				o = &commit->object;
1538
		}
1539

1540
		if (!o)
1541
			o = parse_object_or_die(&oid, refname_nons);
1542

1543
		if (!(o->flags & WANTED)) {
1544
			o->flags |= WANTED;
1545
			add_object_array(o, NULL, want_obj);
1546
		}
1547

1548
		return 1;
1549
	}
1550

1551
	return 0;
1552
}
1553

1554
static int parse_have(const char *line, struct upload_pack_data *data)
1555
{
1556
	const char *arg;
1557
	if (skip_prefix(line, "have ", &arg)) {
1558
		struct object_id oid;
1559

1560
		got_oid(data, arg, &oid);
1561
		data->seen_haves = 1;
1562
		return 1;
1563
	}
1564

1565
	return 0;
1566
}
1567

1568
static void trace2_fetch_info(struct upload_pack_data *data)
1569
{
1570
	struct json_writer jw = JSON_WRITER_INIT;
1571

1572
	jw_object_begin(&jw, 0);
1573
	jw_object_intmax(&jw, "haves", data->have_obj.nr);
1574
	jw_object_intmax(&jw, "wants", data->want_obj.nr);
1575
	jw_object_intmax(&jw, "want-refs", strmap_get_size(&data->wanted_refs));
1576
	jw_object_intmax(&jw, "depth", data->depth);
1577
	jw_object_intmax(&jw, "shallows", data->shallows.nr);
1578
	jw_object_bool(&jw, "deepen-since", data->deepen_since);
1579
	jw_object_intmax(&jw, "deepen-not", oidset_size(&data->deepen_not));
1580
	jw_object_bool(&jw, "deepen-relative", data->deepen_relative);
1581
	if (data->filter_options.choice)
1582
		jw_object_string(&jw, "filter", list_object_filter_config_name(data->filter_options.choice));
1583
	else
1584
		jw_object_null(&jw, "filter");
1585
	jw_end(&jw);
1586

1587
	trace2_data_json("upload-pack", the_repository, "fetch-info", &jw);
1588

1589
	jw_release(&jw);
1590
}
1591

1592
static void process_args(struct packet_reader *request,
1593
			 struct upload_pack_data *data)
1594
{
1595
	while (packet_reader_read(request) == PACKET_READ_NORMAL) {
1596
		const char *arg = request->line;
1597
		const char *p;
1598

1599
		/* process want */
1600
		if (parse_want(&data->writer, arg, &data->want_obj))
1601
			continue;
1602
		if (data->allow_ref_in_want &&
1603
		    parse_want_ref(&data->writer, arg, &data->wanted_refs,
1604
				   &data->hidden_refs, &data->want_obj))
1605
			continue;
1606
		/* process have line */
1607
		if (parse_have(arg, data))
1608
			continue;
1609

1610
		/* process args like thin-pack */
1611
		if (!strcmp(arg, "thin-pack")) {
1612
			data->use_thin_pack = 1;
1613
			continue;
1614
		}
1615
		if (!strcmp(arg, "ofs-delta")) {
1616
			data->use_ofs_delta = 1;
1617
			continue;
1618
		}
1619
		if (!strcmp(arg, "no-progress")) {
1620
			data->no_progress = 1;
1621
			continue;
1622
		}
1623
		if (!strcmp(arg, "include-tag")) {
1624
			data->use_include_tag = 1;
1625
			continue;
1626
		}
1627
		if (!strcmp(arg, "done")) {
1628
			data->done = 1;
1629
			continue;
1630
		}
1631
		if (!strcmp(arg, "wait-for-done")) {
1632
			data->wait_for_done = 1;
1633
			continue;
1634
		}
1635

1636
		/* Shallow related arguments */
1637
		if (process_shallow(arg, &data->shallows))
1638
			continue;
1639
		if (process_deepen(arg, &data->depth))
1640
			continue;
1641
		if (process_deepen_since(arg, &data->deepen_since,
1642
					 &data->deepen_rev_list))
1643
			continue;
1644
		if (process_deepen_not(arg, &data->deepen_not,
1645
				       &data->deepen_rev_list))
1646
			continue;
1647
		if (!strcmp(arg, "deepen-relative")) {
1648
			data->deepen_relative = 1;
1649
			continue;
1650
		}
1651

1652
		if (data->allow_filter && skip_prefix(arg, "filter ", &p)) {
1653
			list_objects_filter_die_if_populated(&data->filter_options);
1654
			parse_list_objects_filter(&data->filter_options, p);
1655
			die_if_using_banned_filter(data);
1656
			continue;
1657
		}
1658

1659
		if (data->allow_sideband_all &&
1660
		    !strcmp(arg, "sideband-all")) {
1661
			data->writer.use_sideband = 1;
1662
			continue;
1663
		}
1664

1665
		if (data->allow_packfile_uris &&
1666
		    skip_prefix(arg, "packfile-uris ", &p)) {
1667
			if (data->uri_protocols.nr)
1668
				send_err_and_die(data,
1669
						 "multiple packfile-uris lines forbidden");
1670
			string_list_split(&data->uri_protocols, p, ',', -1);
1671
			continue;
1672
		}
1673

1674
		/* ignore unknown lines maybe? */
1675
		die("unexpected line: '%s'", arg);
1676
	}
1677

1678
	if (data->uri_protocols.nr && !data->writer.use_sideband)
1679
		string_list_clear(&data->uri_protocols, 0);
1680

1681
	if (request->status != PACKET_READ_FLUSH)
1682
		die(_("expected flush after fetch arguments"));
1683

1684
	if (trace2_is_enabled())
1685
		trace2_fetch_info(data);
1686
}
1687

1688
static int send_acks(struct upload_pack_data *data, struct object_array *acks)
1689
{
1690
	int i;
1691

1692
	packet_writer_write(&data->writer, "acknowledgments\n");
1693

1694
	/* Send Acks */
1695
	if (!acks->nr)
1696
		packet_writer_write(&data->writer, "NAK\n");
1697

1698
	for (i = 0; i < acks->nr; i++) {
1699
		packet_writer_write(&data->writer, "ACK %s\n",
1700
				    oid_to_hex(&acks->objects[i].item->oid));
1701
	}
1702

1703
	if (!data->wait_for_done && ok_to_give_up(data)) {
1704
		/* Send Ready */
1705
		packet_writer_write(&data->writer, "ready\n");
1706
		return 1;
1707
	}
1708

1709
	return 0;
1710
}
1711

1712
static int process_haves_and_send_acks(struct upload_pack_data *data)
1713
{
1714
	int ret = 0;
1715

1716
	if (data->done) {
1717
		ret = 1;
1718
	} else if (send_acks(data, &data->have_obj)) {
1719
		packet_writer_delim(&data->writer);
1720
		ret = 1;
1721
	} else {
1722
		/* Add Flush */
1723
		packet_writer_flush(&data->writer);
1724
		ret = 0;
1725
	}
1726

1727
	return ret;
1728
}
1729

1730
static void send_wanted_ref_info(struct upload_pack_data *data)
1731
{
1732
	struct hashmap_iter iter;
1733
	const struct strmap_entry *e;
1734

1735
	if (strmap_empty(&data->wanted_refs))
1736
		return;
1737

1738
	packet_writer_write(&data->writer, "wanted-refs\n");
1739

1740
	strmap_for_each_entry(&data->wanted_refs, &iter, e) {
1741
		packet_writer_write(&data->writer, "%s %s\n",
1742
				    oid_to_hex(e->value),
1743
				    e->key);
1744
	}
1745

1746
	packet_writer_delim(&data->writer);
1747
}
1748

1749
static void send_shallow_info(struct upload_pack_data *data)
1750
{
1751
	/* No shallow info needs to be sent */
1752
	if (!data->depth && !data->deepen_rev_list && !data->shallows.nr &&
1753
	    !is_repository_shallow(the_repository))
1754
		return;
1755

1756
	packet_writer_write(&data->writer, "shallow-info\n");
1757

1758
	if (!send_shallow_list(data) &&
1759
	    is_repository_shallow(the_repository))
1760
		deepen(data, INFINITE_DEPTH);
1761

1762
	packet_delim(1);
1763
}
1764

1765
enum fetch_state {
1766
	FETCH_PROCESS_ARGS = 0,
1767
	FETCH_SEND_ACKS,
1768
	FETCH_SEND_PACK,
1769
	FETCH_DONE,
1770
};
1771

1772
int upload_pack_v2(struct repository *r, struct packet_reader *request)
1773
{
1774
	enum fetch_state state = FETCH_PROCESS_ARGS;
1775
	struct upload_pack_data data;
1776

1777
	clear_object_flags(ALL_FLAGS);
1778

1779
	upload_pack_data_init(&data);
1780
	data.use_sideband = LARGE_PACKET_MAX;
1781
	get_upload_pack_config(r, &data);
1782

1783
	while (state != FETCH_DONE) {
1784
		switch (state) {
1785
		case FETCH_PROCESS_ARGS:
1786
			process_args(request, &data);
1787

1788
			if (!data.want_obj.nr && !data.wait_for_done) {
1789
				/*
1790
				 * Request didn't contain any 'want' lines (and
1791
				 * the request does not contain
1792
				 * "wait-for-done", in which it is reasonable
1793
				 * to just send 'have's without 'want's); guess
1794
				 * they didn't want anything.
1795
				 */
1796
				state = FETCH_DONE;
1797
			} else if (data.seen_haves) {
1798
				/*
1799
				 * Request had 'have' lines, so lets ACK them.
1800
				 */
1801
				state = FETCH_SEND_ACKS;
1802
			} else {
1803
				/*
1804
				 * Request had 'want's but no 'have's so we can
1805
				 * immedietly go to construct and send a pack.
1806
				 */
1807
				state = FETCH_SEND_PACK;
1808
			}
1809
			break;
1810
		case FETCH_SEND_ACKS:
1811
			if (process_haves_and_send_acks(&data))
1812
				state = FETCH_SEND_PACK;
1813
			else
1814
				state = FETCH_DONE;
1815
			break;
1816
		case FETCH_SEND_PACK:
1817
			send_wanted_ref_info(&data);
1818
			send_shallow_info(&data);
1819

1820
			if (data.uri_protocols.nr) {
1821
				create_pack_file(&data, &data.uri_protocols);
1822
			} else {
1823
				packet_writer_write(&data.writer, "packfile\n");
1824
				create_pack_file(&data, NULL);
1825
			}
1826
			state = FETCH_DONE;
1827
			break;
1828
		case FETCH_DONE:
1829
			continue;
1830
		}
1831
	}
1832

1833
	upload_pack_data_clear(&data);
1834
	return 0;
1835
}
1836

1837
int upload_pack_advertise(struct repository *r,
1838
			  struct strbuf *value)
1839
{
1840
	struct upload_pack_data data;
1841

1842
	upload_pack_data_init(&data);
1843
	get_upload_pack_config(r, &data);
1844

1845
	if (value) {
1846
		strbuf_addstr(value, "shallow wait-for-done");
1847

1848
		if (data.allow_filter)
1849
			strbuf_addstr(value, " filter");
1850

1851
		if (data.allow_ref_in_want)
1852
			strbuf_addstr(value, " ref-in-want");
1853

1854
		if (data.allow_sideband_all)
1855
			strbuf_addstr(value, " sideband-all");
1856

1857
		if (data.allow_packfile_uris)
1858
			strbuf_addstr(value, " packfile-uris");
1859
	}
1860

1861
	upload_pack_data_clear(&data);
1862

1863
	return 1;
1864
}
1865