git

1
#define USE_THE_REPOSITORY_VARIABLE
2

3
#include "git-compat-util.h"
4
#include "repository.h"
5
#include "hex.h"
6
#include "walker.h"
7
#include "http.h"
8
#include "list.h"
9
#include "transport.h"
10
#include "packfile.h"
11
#include "object-store-ll.h"
12

13
struct alt_base {
14
	char *base;
15
	int got_indices;
16
	struct packed_git *packs;
17
	struct alt_base *next;
18
};
19

20
enum object_request_state {
21
	WAITING,
22
	ABORTED,
23
	ACTIVE,
24
	COMPLETE
25
};
26

27
struct object_request {
28
	struct walker *walker;
29
	struct object_id oid;
30
	struct alt_base *repo;
31
	enum object_request_state state;
32
	struct http_object_request *req;
33
	struct list_head node;
34
};
35

36
struct alternates_request {
37
	struct walker *walker;
38
	const char *base;
39
	struct strbuf *url;
40
	struct strbuf *buffer;
41
	struct active_request_slot *slot;
42
	int http_specific;
43
};
44

45
struct walker_data {
46
	const char *url;
47
	int got_alternates;
48
	struct alt_base *alt;
49
};
50

51
static LIST_HEAD(object_queue_head);
52

53
static void fetch_alternates(struct walker *walker, const char *base);
54

55
static void process_object_response(void *callback_data);
56

57
static void start_object_request(struct object_request *obj_req)
58
{
59
	struct active_request_slot *slot;
60
	struct http_object_request *req;
61

62
	req = new_http_object_request(obj_req->repo->base, &obj_req->oid);
63
	if (!req) {
64
		obj_req->state = ABORTED;
65
		return;
66
	}
67
	obj_req->req = req;
68

69
	slot = req->slot;
70
	slot->callback_func = process_object_response;
71
	slot->callback_data = obj_req;
72

73
	/* Try to get the request started, abort the request on error */
74
	obj_req->state = ACTIVE;
75
	if (!start_active_slot(slot)) {
76
		obj_req->state = ABORTED;
77
		release_http_object_request(req);
78
		return;
79
	}
80
}
81

82
static void finish_object_request(struct object_request *obj_req)
83
{
84
	if (finish_http_object_request(obj_req->req))
85
		return;
86

87
	if (obj_req->req->rename == 0)
88
		walker_say(obj_req->walker, "got %s\n", oid_to_hex(&obj_req->oid));
89
}
90

91
static void process_object_response(void *callback_data)
92
{
93
	struct object_request *obj_req =
94
		(struct object_request *)callback_data;
95
	struct walker *walker = obj_req->walker;
96
	struct walker_data *data = walker->data;
97
	struct alt_base *alt = data->alt;
98

99
	process_http_object_request(obj_req->req);
100
	obj_req->state = COMPLETE;
101

102
	normalize_curl_result(&obj_req->req->curl_result,
103
			      obj_req->req->http_code,
104
			      obj_req->req->errorstr,
105
			      sizeof(obj_req->req->errorstr));
106

107
	/* Use alternates if necessary */
108
	if (missing_target(obj_req->req)) {
109
		fetch_alternates(walker, alt->base);
110
		if (obj_req->repo->next) {
111
			obj_req->repo =
112
				obj_req->repo->next;
113
			release_http_object_request(obj_req->req);
114
			start_object_request(obj_req);
115
			return;
116
		}
117
	}
118

119
	finish_object_request(obj_req);
120
}
121

122
static void release_object_request(struct object_request *obj_req)
123
{
124
	if (obj_req->req !=NULL && obj_req->req->localfile != -1)
125
		error("fd leakage in release: %d", obj_req->req->localfile);
126

127
	list_del(&obj_req->node);
128
	free(obj_req);
129
}
130

131
static int fill_active_slot(void *data UNUSED)
132
{
133
	struct object_request *obj_req;
134
	struct list_head *pos, *tmp, *head = &object_queue_head;
135

136
	list_for_each_safe(pos, tmp, head) {
137
		obj_req = list_entry(pos, struct object_request, node);
138
		if (obj_req->state == WAITING) {
139
			if (repo_has_object_file(the_repository, &obj_req->oid))
140
				obj_req->state = COMPLETE;
141
			else {
142
				start_object_request(obj_req);
143
				return 1;
144
			}
145
		}
146
	}
147
	return 0;
148
}
149

150
static void prefetch(struct walker *walker, unsigned char *sha1)
151
{
152
	struct object_request *newreq;
153
	struct walker_data *data = walker->data;
154

155
	newreq = xmalloc(sizeof(*newreq));
156
	newreq->walker = walker;
157
	oidread(&newreq->oid, sha1, the_repository->hash_algo);
158
	newreq->repo = data->alt;
159
	newreq->state = WAITING;
160
	newreq->req = NULL;
161

162
	http_is_verbose = walker->get_verbosely;
163
	list_add_tail(&newreq->node, &object_queue_head);
164

165
	fill_active_slots();
166
	step_active_slots();
167
}
168

169
static int is_alternate_allowed(const char *url)
170
{
171
	const char *protocols[] = {
172
		"http", "https", "ftp", "ftps"
173
	};
174
	int i;
175

176
	if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
177
		warning("alternate disabled by http.followRedirects: %s", url);
178
		return 0;
179
	}
180

181
	for (i = 0; i < ARRAY_SIZE(protocols); i++) {
182
		const char *end;
183
		if (skip_prefix(url, protocols[i], &end) &&
184
		    starts_with(end, "://"))
185
			break;
186
	}
187

188
	if (i >= ARRAY_SIZE(protocols)) {
189
		warning("ignoring alternate with unknown protocol: %s", url);
190
		return 0;
191
	}
192
	if (!is_transport_allowed(protocols[i], 0)) {
193
		warning("ignoring alternate with restricted protocol: %s", url);
194
		return 0;
195
	}
196

197
	return 1;
198
}
199

200
static void process_alternates_response(void *callback_data)
201
{
202
	struct alternates_request *alt_req =
203
		(struct alternates_request *)callback_data;
204
	struct walker *walker = alt_req->walker;
205
	struct walker_data *cdata = walker->data;
206
	struct active_request_slot *slot = alt_req->slot;
207
	struct alt_base *tail = cdata->alt;
208
	const char *base = alt_req->base;
209
	const char null_byte = '\0';
210
	char *data;
211
	int i = 0;
212

213
	normalize_curl_result(&slot->curl_result, slot->http_code,
214
			      curl_errorstr, sizeof(curl_errorstr));
215

216
	if (alt_req->http_specific) {
217
		if (slot->curl_result != CURLE_OK ||
218
		    !alt_req->buffer->len) {
219

220
			/* Try reusing the slot to get non-http alternates */
221
			alt_req->http_specific = 0;
222
			strbuf_reset(alt_req->url);
223
			strbuf_addf(alt_req->url, "%s/objects/info/alternates",
224
				    base);
225
			curl_easy_setopt(slot->curl, CURLOPT_URL,
226
					 alt_req->url->buf);
227
			active_requests++;
228
			slot->in_use = 1;
229
			if (slot->finished)
230
				(*slot->finished) = 0;
231
			if (!start_active_slot(slot)) {
232
				cdata->got_alternates = -1;
233
				slot->in_use = 0;
234
				if (slot->finished)
235
					(*slot->finished) = 1;
236
			}
237
			return;
238
		}
239
	} else if (slot->curl_result != CURLE_OK) {
240
		if (!missing_target(slot)) {
241
			cdata->got_alternates = -1;
242
			return;
243
		}
244
	}
245

246
	fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
247
	alt_req->buffer->len--;
248
	data = alt_req->buffer->buf;
249

250
	while (i < alt_req->buffer->len) {
251
		int posn = i;
252
		while (posn < alt_req->buffer->len && data[posn] != '\n')
253
			posn++;
254
		if (data[posn] == '\n') {
255
			int okay = 0;
256
			int serverlen = 0;
257
			struct alt_base *newalt;
258
			if (data[i] == '/') {
259
				/*
260
				 * This counts
261
				 * http://git.host/pub/scm/linux.git/
262
				 * -----------here^
263
				 * so memcpy(dst, base, serverlen) will
264
				 * copy up to "...git.host".
265
				 */
266
				const char *colon_ss = strstr(base,"://");
267
				if (colon_ss) {
268
					serverlen = (strchr(colon_ss + 3, '/')
269
						     - base);
270
					okay = 1;
271
				}
272
			} else if (!memcmp(data + i, "../", 3)) {
273
				/*
274
				 * Relative URL; chop the corresponding
275
				 * number of subpath from base (and ../
276
				 * from data), and concatenate the result.
277
				 *
278
				 * The code first drops ../ from data, and
279
				 * then drops one ../ from data and one path
280
				 * from base.  IOW, one extra ../ is dropped
281
				 * from data than path is dropped from base.
282
				 *
283
				 * This is not wrong.  The alternate in
284
				 *     http://git.host/pub/scm/linux.git/
285
				 * to borrow from
286
				 *     http://git.host/pub/scm/linus.git/
287
				 * is ../../linus.git/objects/.  You need
288
				 * two ../../ to borrow from your direct
289
				 * neighbour.
290
				 */
291
				i += 3;
292
				serverlen = strlen(base);
293
				while (i + 2 < posn &&
294
				       !memcmp(data + i, "../", 3)) {
295
					do {
296
						serverlen--;
297
					} while (serverlen &&
298
						 base[serverlen - 1] != '/');
299
					i += 3;
300
				}
301
				/* If the server got removed, give up. */
302
				okay = strchr(base, ':') - base + 3 <
303
				       serverlen;
304
			} else if (alt_req->http_specific) {
305
				char *colon = strchr(data + i, ':');
306
				char *slash = strchr(data + i, '/');
307
				if (colon && slash && colon < data + posn &&
308
				    slash < data + posn && colon < slash) {
309
					okay = 1;
310
				}
311
			}
312
			if (okay) {
313
				struct strbuf target = STRBUF_INIT;
314
				strbuf_add(&target, base, serverlen);
315
				strbuf_add(&target, data + i, posn - i);
316
				if (!strbuf_strip_suffix(&target, "objects")) {
317
					warning("ignoring alternate that does"
318
						" not end in 'objects': %s",
319
						target.buf);
320
					strbuf_release(&target);
321
				} else if (is_alternate_allowed(target.buf)) {
322
					warning("adding alternate object store: %s",
323
						target.buf);
324
					newalt = xmalloc(sizeof(*newalt));
325
					newalt->next = NULL;
326
					newalt->base = strbuf_detach(&target, NULL);
327
					newalt->got_indices = 0;
328
					newalt->packs = NULL;
329

330
					while (tail->next != NULL)
331
						tail = tail->next;
332
					tail->next = newalt;
333
				} else {
334
					strbuf_release(&target);
335
				}
336
			}
337
		}
338
		i = posn + 1;
339
	}
340

341
	cdata->got_alternates = 1;
342
}
343

344
static void fetch_alternates(struct walker *walker, const char *base)
345
{
346
	struct strbuf buffer = STRBUF_INIT;
347
	struct strbuf url = STRBUF_INIT;
348
	struct active_request_slot *slot;
349
	struct alternates_request alt_req;
350
	struct walker_data *cdata = walker->data;
351

352
	/*
353
	 * If another request has already started fetching alternates,
354
	 * wait for them to arrive and return to processing this request's
355
	 * curl message
356
	 */
357
	while (cdata->got_alternates == 0) {
358
		step_active_slots();
359
	}
360

361
	/* Nothing to do if they've already been fetched */
362
	if (cdata->got_alternates == 1)
363
		return;
364

365
	/* Start the fetch */
366
	cdata->got_alternates = 0;
367

368
	if (walker->get_verbosely)
369
		fprintf(stderr, "Getting alternates list for %s\n", base);
370

371
	strbuf_addf(&url, "%s/objects/info/http-alternates", base);
372

373
	/*
374
	 * Use a callback to process the result, since another request
375
	 * may fail and need to have alternates loaded before continuing
376
	 */
377
	slot = get_active_slot();
378
	slot->callback_func = process_alternates_response;
379
	alt_req.walker = walker;
380
	slot->callback_data = &alt_req;
381

382
	curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, &buffer);
383
	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
384
	curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
385

386
	alt_req.base = base;
387
	alt_req.url = &url;
388
	alt_req.buffer = &buffer;
389
	alt_req.http_specific = 1;
390
	alt_req.slot = slot;
391

392
	if (start_active_slot(slot))
393
		run_active_slot(slot);
394
	else
395
		cdata->got_alternates = -1;
396

397
	strbuf_release(&buffer);
398
	strbuf_release(&url);
399
}
400

401
static int fetch_indices(struct walker *walker, struct alt_base *repo)
402
{
403
	int ret;
404

405
	if (repo->got_indices)
406
		return 0;
407

408
	if (walker->get_verbosely)
409
		fprintf(stderr, "Getting pack list for %s\n", repo->base);
410

411
	switch (http_get_info_packs(repo->base, &repo->packs)) {
412
	case HTTP_OK:
413
	case HTTP_MISSING_TARGET:
414
		repo->got_indices = 1;
415
		ret = 0;
416
		break;
417
	default:
418
		repo->got_indices = 0;
419
		ret = -1;
420
	}
421

422
	return ret;
423
}
424

425
static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
426
{
427
	struct packed_git *target;
428
	int ret;
429
	struct slot_results results;
430
	struct http_pack_request *preq;
431

432
	if (fetch_indices(walker, repo))
433
		return -1;
434
	target = find_sha1_pack(sha1, repo->packs);
435
	if (!target)
436
		return -1;
437
	close_pack_index(target);
438

439
	if (walker->get_verbosely) {
440
		fprintf(stderr, "Getting pack %s\n",
441
			hash_to_hex(target->hash));
442
		fprintf(stderr, " which contains %s\n",
443
			hash_to_hex(sha1));
444
	}
445

446
	preq = new_http_pack_request(target->hash, repo->base);
447
	if (!preq)
448
		goto abort;
449
	preq->slot->results = &results;
450

451
	if (start_active_slot(preq->slot)) {
452
		run_active_slot(preq->slot);
453
		if (results.curl_result != CURLE_OK) {
454
			error("Unable to get pack file %s\n%s", preq->url,
455
			      curl_errorstr);
456
			goto abort;
457
		}
458
	} else {
459
		error("Unable to start request");
460
		goto abort;
461
	}
462

463
	ret = finish_http_pack_request(preq);
464
	release_http_pack_request(preq);
465
	if (ret)
466
		return ret;
467
	http_install_packfile(target, &repo->packs);
468

469
	return 0;
470

471
abort:
472
	return -1;
473
}
474

475
static void abort_object_request(struct object_request *obj_req)
476
{
477
	release_object_request(obj_req);
478
}
479

480
static int fetch_object(struct walker *walker, unsigned char *hash)
481
{
482
	char *hex = hash_to_hex(hash);
483
	int ret = 0;
484
	struct object_request *obj_req = NULL;
485
	struct http_object_request *req;
486
	struct list_head *pos, *head = &object_queue_head;
487

488
	list_for_each(pos, head) {
489
		obj_req = list_entry(pos, struct object_request, node);
490
		if (hasheq(obj_req->oid.hash, hash, the_repository->hash_algo))
491
			break;
492
	}
493
	if (!obj_req)
494
		return error("Couldn't find request for %s in the queue", hex);
495

496
	if (repo_has_object_file(the_repository, &obj_req->oid)) {
497
		if (obj_req->req)
498
			abort_http_object_request(obj_req->req);
499
		abort_object_request(obj_req);
500
		return 0;
501
	}
502

503
	while (obj_req->state == WAITING)
504
		step_active_slots();
505

506
	/*
507
	 * obj_req->req might change when fetching alternates in the callback
508
	 * process_object_response; therefore, the "shortcut" variable, req,
509
	 * is used only after we're done with slots.
510
	 */
511
	while (obj_req->state == ACTIVE)
512
		run_active_slot(obj_req->req->slot);
513

514
	req = obj_req->req;
515

516
	if (req->localfile != -1) {
517
		close(req->localfile);
518
		req->localfile = -1;
519
	}
520

521
	normalize_curl_result(&req->curl_result, req->http_code,
522
			      req->errorstr, sizeof(req->errorstr));
523

524
	if (obj_req->state == ABORTED) {
525
		ret = error("Request for %s aborted", hex);
526
	} else if (req->curl_result != CURLE_OK &&
527
		   req->http_code != 416) {
528
		if (missing_target(req))
529
			ret = -1; /* Be silent, it is probably in a pack. */
530
		else
531
			ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
532
				    req->errorstr, req->curl_result,
533
				    req->http_code, hex);
534
	} else if (req->zret != Z_STREAM_END) {
535
		walker->corrupt_object_found++;
536
		ret = error("File %s (%s) corrupt", hex, req->url);
537
	} else if (!oideq(&obj_req->oid, &req->real_oid)) {
538
		ret = error("File %s has bad hash", hex);
539
	} else if (req->rename < 0) {
540
		struct strbuf buf = STRBUF_INIT;
541
		loose_object_path(the_repository, &buf, &req->oid);
542
		ret = error("unable to write sha1 filename %s", buf.buf);
543
		strbuf_release(&buf);
544
	}
545

546
	release_http_object_request(req);
547
	release_object_request(obj_req);
548
	return ret;
549
}
550

551
static int fetch(struct walker *walker, unsigned char *hash)
552
{
553
	struct walker_data *data = walker->data;
554
	struct alt_base *altbase = data->alt;
555

556
	if (!fetch_object(walker, hash))
557
		return 0;
558
	while (altbase) {
559
		if (!http_fetch_pack(walker, altbase, hash))
560
			return 0;
561
		fetch_alternates(walker, data->alt->base);
562
		altbase = altbase->next;
563
	}
564
	return error("Unable to find %s under %s", hash_to_hex(hash),
565
		     data->alt->base);
566
}
567

568
static int fetch_ref(struct walker *walker, struct ref *ref)
569
{
570
	struct walker_data *data = walker->data;
571
	return http_fetch_ref(data->alt->base, ref);
572
}
573

574
static void cleanup(struct walker *walker)
575
{
576
	struct walker_data *data = walker->data;
577
	struct alt_base *alt, *alt_next;
578

579
	if (data) {
580
		alt = data->alt;
581
		while (alt) {
582
			alt_next = alt->next;
583

584
			free(alt->base);
585
			free(alt);
586

587
			alt = alt_next;
588
		}
589
		free(data);
590
		walker->data = NULL;
591
	}
592
}
593

594
struct walker *get_http_walker(const char *url)
595
{
596
	char *s;
597
	struct walker_data *data = xmalloc(sizeof(struct walker_data));
598
	struct walker *walker = xmalloc(sizeof(struct walker));
599

600
	data->alt = xmalloc(sizeof(*data->alt));
601
	data->alt->base = xstrdup(url);
602
	for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
603
		*s = 0;
604

605
	data->alt->got_indices = 0;
606
	data->alt->packs = NULL;
607
	data->alt->next = NULL;
608
	data->got_alternates = -1;
609

610
	walker->corrupt_object_found = 0;
611
	walker->fetch = fetch;
612
	walker->fetch_ref = fetch_ref;
613
	walker->prefetch = prefetch;
614
	walker->cleanup = cleanup;
615
	walker->data = data;
616

617
	add_fill_function(NULL, fill_active_slot);
618

619
	return walker;
620
}
621