git

1
#define USE_THE_REPOSITORY_VARIABLE
2

3
#include "git-compat-util.h"
4
#include "advice.h"
5
#include "config.h"
6
#include "convert.h"
7
#include "copy.h"
8
#include "gettext.h"
9
#include "hex.h"
10
#include "object-store-ll.h"
11
#include "attr.h"
12
#include "run-command.h"
13
#include "quote.h"
14
#include "read-cache-ll.h"
15
#include "sigchain.h"
16
#include "pkt-line.h"
17
#include "sub-process.h"
18
#include "trace.h"
19
#include "utf8.h"
20
#include "merge-ll.h"
21

22
/*
23
 * convert.c - convert a file when checking it out and checking it in.
24
 *
25
 * This should use the pathname to decide on whether it wants to do some
26
 * more interesting conversions (automatic gzip/unzip, general format
27
 * conversions etc etc), but by default it just does automatic CRLF<->LF
28
 * translation when the "text" attribute or "auto_crlf" option is set.
29
 */
30

31
/* Stat bits: When BIN is set, the txt bits are unset */
32
#define CONVERT_STAT_BITS_TXT_LF    0x1
33
#define CONVERT_STAT_BITS_TXT_CRLF  0x2
34
#define CONVERT_STAT_BITS_BIN       0x4
35

36
struct text_stat {
37
	/* NUL, CR, LF and CRLF counts */
38
	unsigned nul, lonecr, lonelf, crlf;
39

40
	/* These are just approximations! */
41
	unsigned printable, nonprintable;
42
};
43

44
static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
45
{
46
	unsigned long i;
47

48
	memset(stats, 0, sizeof(*stats));
49

50
	for (i = 0; i < size; i++) {
51
		unsigned char c = buf[i];
52
		if (c == '\r') {
53
			if (i+1 < size && buf[i+1] == '\n') {
54
				stats->crlf++;
55
				i++;
56
			} else
57
				stats->lonecr++;
58
			continue;
59
		}
60
		if (c == '\n') {
61
			stats->lonelf++;
62
			continue;
63
		}
64
		if (c == 127)
65
			/* DEL */
66
			stats->nonprintable++;
67
		else if (c < 32) {
68
			switch (c) {
69
				/* BS, HT, ESC and FF */
70
			case '\b': case '\t': case '\033': case '\014':
71
				stats->printable++;
72
				break;
73
			case 0:
74
				stats->nul++;
75
				/* fall through */
76
			default:
77
				stats->nonprintable++;
78
			}
79
		}
80
		else
81
			stats->printable++;
82
	}
83

84
	/* If file ends with EOF then don't count this EOF as non-printable. */
85
	if (size >= 1 && buf[size-1] == '\032')
86
		stats->nonprintable--;
87
}
88

89
/*
90
 * The same heuristics as diff.c::mmfile_is_binary()
91
 * We treat files with bare CR as binary
92
 */
93
static int convert_is_binary(const struct text_stat *stats)
94
{
95
	if (stats->lonecr)
96
		return 1;
97
	if (stats->nul)
98
		return 1;
99
	if ((stats->printable >> 7) < stats->nonprintable)
100
		return 1;
101
	return 0;
102
}
103

104
static unsigned int gather_convert_stats(const char *data, unsigned long size)
105
{
106
	struct text_stat stats;
107
	int ret = 0;
108
	if (!data || !size)
109
		return 0;
110
	gather_stats(data, size, &stats);
111
	if (convert_is_binary(&stats))
112
		ret |= CONVERT_STAT_BITS_BIN;
113
	if (stats.crlf)
114
		ret |= CONVERT_STAT_BITS_TXT_CRLF;
115
	if (stats.lonelf)
116
		ret |=  CONVERT_STAT_BITS_TXT_LF;
117

118
	return ret;
119
}
120

121
static const char *gather_convert_stats_ascii(const char *data, unsigned long size)
122
{
123
	unsigned int convert_stats = gather_convert_stats(data, size);
124

125
	if (convert_stats & CONVERT_STAT_BITS_BIN)
126
		return "-text";
127
	switch (convert_stats) {
128
	case CONVERT_STAT_BITS_TXT_LF:
129
		return "lf";
130
	case CONVERT_STAT_BITS_TXT_CRLF:
131
		return "crlf";
132
	case CONVERT_STAT_BITS_TXT_LF | CONVERT_STAT_BITS_TXT_CRLF:
133
		return "mixed";
134
	default:
135
		return "none";
136
	}
137
}
138

139
const char *get_cached_convert_stats_ascii(struct index_state *istate,
140
					   const char *path)
141
{
142
	const char *ret;
143
	unsigned long sz;
144
	void *data = read_blob_data_from_index(istate, path, &sz);
145
	ret = gather_convert_stats_ascii(data, sz);
146
	free(data);
147
	return ret;
148
}
149

150
const char *get_wt_convert_stats_ascii(const char *path)
151
{
152
	const char *ret = "";
153
	struct strbuf sb = STRBUF_INIT;
154
	if (strbuf_read_file(&sb, path, 0) >= 0)
155
		ret = gather_convert_stats_ascii(sb.buf, sb.len);
156
	strbuf_release(&sb);
157
	return ret;
158
}
159

160
static int text_eol_is_crlf(void)
161
{
162
	if (auto_crlf == AUTO_CRLF_TRUE)
163
		return 1;
164
	else if (auto_crlf == AUTO_CRLF_INPUT)
165
		return 0;
166
	if (core_eol == EOL_CRLF)
167
		return 1;
168
	if (core_eol == EOL_UNSET && EOL_NATIVE == EOL_CRLF)
169
		return 1;
170
	return 0;
171
}
172

173
static enum eol output_eol(enum convert_crlf_action crlf_action)
174
{
175
	switch (crlf_action) {
176
	case CRLF_BINARY:
177
		return EOL_UNSET;
178
	case CRLF_TEXT_CRLF:
179
		return EOL_CRLF;
180
	case CRLF_TEXT_INPUT:
181
		return EOL_LF;
182
	case CRLF_UNDEFINED:
183
	case CRLF_AUTO_CRLF:
184
		return EOL_CRLF;
185
	case CRLF_AUTO_INPUT:
186
		return EOL_LF;
187
	case CRLF_TEXT:
188
	case CRLF_AUTO:
189
		/* fall through */
190
		return text_eol_is_crlf() ? EOL_CRLF : EOL_LF;
191
	}
192
	warning(_("illegal crlf_action %d"), (int)crlf_action);
193
	return core_eol;
194
}
195

196
static void check_global_conv_flags_eol(const char *path,
197
			    struct text_stat *old_stats, struct text_stat *new_stats,
198
			    int conv_flags)
199
{
200
	if (old_stats->crlf && !new_stats->crlf ) {
201
		/*
202
		 * CRLFs would not be restored by checkout
203
		 */
204
		if (conv_flags & CONV_EOL_RNDTRP_DIE)
205
			die(_("CRLF would be replaced by LF in %s"), path);
206
		else if (conv_flags & CONV_EOL_RNDTRP_WARN)
207
			warning(_("in the working copy of '%s', CRLF will be"
208
				  " replaced by LF the next time Git touches"
209
				  " it"), path);
210
	} else if (old_stats->lonelf && !new_stats->lonelf ) {
211
		/*
212
		 * CRLFs would be added by checkout
213
		 */
214
		if (conv_flags & CONV_EOL_RNDTRP_DIE)
215
			die(_("LF would be replaced by CRLF in %s"), path);
216
		else if (conv_flags & CONV_EOL_RNDTRP_WARN)
217
			warning(_("in the working copy of '%s', LF will be"
218
				  " replaced by CRLF the next time Git touches"
219
				  " it"), path);
220
	}
221
}
222

223
static int has_crlf_in_index(struct index_state *istate, const char *path)
224
{
225
	unsigned long sz;
226
	void *data;
227
	const char *crp;
228
	int has_crlf = 0;
229

230
	data = read_blob_data_from_index(istate, path, &sz);
231
	if (!data)
232
		return 0;
233

234
	crp = memchr(data, '\r', sz);
235
	if (crp) {
236
		unsigned int ret_stats;
237
		ret_stats = gather_convert_stats(data, sz);
238
		if (!(ret_stats & CONVERT_STAT_BITS_BIN) &&
239
		    (ret_stats & CONVERT_STAT_BITS_TXT_CRLF))
240
			has_crlf = 1;
241
	}
242
	free(data);
243
	return has_crlf;
244
}
245

246
static int will_convert_lf_to_crlf(struct text_stat *stats,
247
				   enum convert_crlf_action crlf_action)
248
{
249
	if (output_eol(crlf_action) != EOL_CRLF)
250
		return 0;
251
	/* No "naked" LF? Nothing to convert, regardless. */
252
	if (!stats->lonelf)
253
		return 0;
254

255
	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
256
		/* If we have any CR or CRLF line endings, we do not touch it */
257
		/* This is the new safer autocrlf-handling */
258
		if (stats->lonecr || stats->crlf)
259
			return 0;
260

261
		if (convert_is_binary(stats))
262
			return 0;
263
	}
264
	return 1;
265

266
}
267

268
static int validate_encoding(const char *path, const char *enc,
269
		      const char *data, size_t len, int die_on_error)
270
{
271
	const char *stripped;
272

273
	/* We only check for UTF here as UTF?? can be an alias for UTF-?? */
274
	if (skip_iprefix(enc, "UTF", &stripped)) {
275
		skip_prefix(stripped, "-", &stripped);
276

277
		/*
278
		 * Check for detectable errors in UTF encodings
279
		 */
280
		if (has_prohibited_utf_bom(enc, data, len)) {
281
			const char *error_msg = _(
282
				"BOM is prohibited in '%s' if encoded as %s");
283
			/*
284
			 * This advice is shown for UTF-??BE and UTF-??LE encodings.
285
			 * We cut off the last two characters of the encoding name
286
			 * to generate the encoding name suitable for BOMs.
287
			 */
288
			const char *advise_msg = _(
289
				"The file '%s' contains a byte order "
290
				"mark (BOM). Please use UTF-%.*s as "
291
				"working-tree-encoding.");
292
			int stripped_len = strlen(stripped) - strlen("BE");
293
			advise(advise_msg, path, stripped_len, stripped);
294
			if (die_on_error)
295
				die(error_msg, path, enc);
296
			else {
297
				return error(error_msg, path, enc);
298
			}
299

300
		} else if (is_missing_required_utf_bom(enc, data, len)) {
301
			const char *error_msg = _(
302
				"BOM is required in '%s' if encoded as %s");
303
			const char *advise_msg = _(
304
				"The file '%s' is missing a byte order "
305
				"mark (BOM). Please use UTF-%sBE or UTF-%sLE "
306
				"(depending on the byte order) as "
307
				"working-tree-encoding.");
308
			advise(advise_msg, path, stripped, stripped);
309
			if (die_on_error)
310
				die(error_msg, path, enc);
311
			else {
312
				return error(error_msg, path, enc);
313
			}
314
		}
315

316
	}
317
	return 0;
318
}
319

320
static void trace_encoding(const char *context, const char *path,
321
			   const char *encoding, const char *buf, size_t len)
322
{
323
	static struct trace_key coe = TRACE_KEY_INIT(WORKING_TREE_ENCODING);
324
	struct strbuf trace = STRBUF_INIT;
325
	int i;
326

327
	if (!trace_want(&coe))
328
		return;
329

330
	strbuf_addf(&trace, "%s (%s, considered %s):\n", context, path, encoding);
331
	for (i = 0; i < len && buf; ++i) {
332
		strbuf_addf(
333
			&trace, "| \033[2m%2i:\033[0m %2x \033[2m%c\033[0m%c",
334
			i,
335
			(unsigned char) buf[i],
336
			(buf[i] > 32 && buf[i] < 127 ? buf[i] : ' '),
337
			((i+1) % 8 && (i+1) < len ? ' ' : '\n')
338
		);
339
	}
340
	strbuf_addchars(&trace, '\n', 1);
341

342
	trace_strbuf(&coe, &trace);
343
	strbuf_release(&trace);
344
}
345

346
static int check_roundtrip(const char *enc_name)
347
{
348
	/*
349
	 * check_roundtrip_encoding contains a string of comma and/or
350
	 * space separated encodings (eg. "UTF-16, ASCII, CP1125").
351
	 * Search for the given encoding in that string.
352
	 */
353
	const char *encoding = check_roundtrip_encoding ?
354
		check_roundtrip_encoding : "SHIFT-JIS";
355
	const char *found = strcasestr(encoding, enc_name);
356
	const char *next;
357
	int len;
358
	if (!found)
359
		return 0;
360
	next = found + strlen(enc_name);
361
	len = strlen(encoding);
362
	return (found && (
363
			/*
364
			 * Check that the found encoding is at the beginning of
365
			 * encoding or that it is prefixed with a space or
366
			 * comma.
367
			 */
368
			found == encoding || (
369
				(isspace(found[-1]) || found[-1] == ',')
370
			)
371
		) && (
372
			/*
373
			 * Check that the found encoding is at the end of
374
			 * encoding or that it is suffixed with a space
375
			 * or comma.
376
			 */
377
			next == encoding + len || (
378
				next < encoding + len &&
379
				(isspace(next[0]) || next[0] == ',')
380
			)
381
		));
382
}
383

384
static const char *default_encoding = "UTF-8";
385

386
static int encode_to_git(const char *path, const char *src, size_t src_len,
387
			 struct strbuf *buf, const char *enc, int conv_flags)
388
{
389
	char *dst;
390
	size_t dst_len;
391
	int die_on_error = conv_flags & CONV_WRITE_OBJECT;
392

393
	/*
394
	 * No encoding is specified or there is nothing to encode.
395
	 * Tell the caller that the content was not modified.
396
	 */
397
	if (!enc || (src && !src_len))
398
		return 0;
399

400
	/*
401
	 * Looks like we got called from "would_convert_to_git()".
402
	 * This means Git wants to know if it would encode (= modify!)
403
	 * the content. Let's answer with "yes", since an encoding was
404
	 * specified.
405
	 */
406
	if (!buf && !src)
407
		return 1;
408

409
	if (validate_encoding(path, enc, src, src_len, die_on_error))
410
		return 0;
411

412
	trace_encoding("source", path, enc, src, src_len);
413
	dst = reencode_string_len(src, src_len, default_encoding, enc,
414
				  &dst_len);
415
	if (!dst) {
416
		/*
417
		 * We could add the blob "as-is" to Git. However, on checkout
418
		 * we would try to re-encode to the original encoding. This
419
		 * would fail and we would leave the user with a messed-up
420
		 * working tree. Let's try to avoid this by screaming loud.
421
		 */
422
		const char* msg = _("failed to encode '%s' from %s to %s");
423
		if (die_on_error)
424
			die(msg, path, enc, default_encoding);
425
		else {
426
			error(msg, path, enc, default_encoding);
427
			return 0;
428
		}
429
	}
430
	trace_encoding("destination", path, default_encoding, dst, dst_len);
431

432
	/*
433
	 * UTF supports lossless conversion round tripping [1] and conversions
434
	 * between UTF and other encodings are mostly round trip safe as
435
	 * Unicode aims to be a superset of all other character encodings.
436
	 * However, certain encodings (e.g. SHIFT-JIS) are known to have round
437
	 * trip issues [2]. Check the round trip conversion for all encodings
438
	 * listed in core.checkRoundtripEncoding.
439
	 *
440
	 * The round trip check is only performed if content is written to Git.
441
	 * This ensures that no information is lost during conversion to/from
442
	 * the internal UTF-8 representation.
443
	 *
444
	 * Please note, the code below is not tested because I was not able to
445
	 * generate a faulty round trip without an iconv error. Iconv errors
446
	 * are already caught above.
447
	 *
448
	 * [1] http://unicode.org/faq/utf_bom.html#gen2
449
	 * [2] https://support.microsoft.com/en-us/help/170559/prb-conversion-problem-between-shift-jis-and-unicode
450
	 */
451
	if (die_on_error && check_roundtrip(enc)) {
452
		char *re_src;
453
		size_t re_src_len;
454

455
		re_src = reencode_string_len(dst, dst_len,
456
					     enc, default_encoding,
457
					     &re_src_len);
458

459
		trace_printf("Checking roundtrip encoding for %s...\n", enc);
460
		trace_encoding("reencoded source", path, enc,
461
			       re_src, re_src_len);
462

463
		if (!re_src || src_len != re_src_len ||
464
		    memcmp(src, re_src, src_len)) {
465
			const char* msg = _("encoding '%s' from %s to %s and "
466
					    "back is not the same");
467
			die(msg, path, enc, default_encoding);
468
		}
469

470
		free(re_src);
471
	}
472

473
	strbuf_attach(buf, dst, dst_len, dst_len + 1);
474
	return 1;
475
}
476

477
static int encode_to_worktree(const char *path, const char *src, size_t src_len,
478
			      struct strbuf *buf, const char *enc)
479
{
480
	char *dst;
481
	size_t dst_len;
482

483
	/*
484
	 * No encoding is specified or there is nothing to encode.
485
	 * Tell the caller that the content was not modified.
486
	 */
487
	if (!enc || (src && !src_len))
488
		return 0;
489

490
	dst = reencode_string_len(src, src_len, enc, default_encoding,
491
				  &dst_len);
492
	if (!dst) {
493
		error(_("failed to encode '%s' from %s to %s"),
494
		      path, default_encoding, enc);
495
		return 0;
496
	}
497

498
	strbuf_attach(buf, dst, dst_len, dst_len + 1);
499
	return 1;
500
}
501

502
static int crlf_to_git(struct index_state *istate,
503
		       const char *path, const char *src, size_t len,
504
		       struct strbuf *buf,
505
		       enum convert_crlf_action crlf_action, int conv_flags)
506
{
507
	struct text_stat stats;
508
	char *dst;
509
	int convert_crlf_into_lf;
510

511
	if (crlf_action == CRLF_BINARY ||
512
	    (src && !len))
513
		return 0;
514

515
	/*
516
	 * If we are doing a dry-run and have no source buffer, there is
517
	 * nothing to analyze; we must assume we would convert.
518
	 */
519
	if (!buf && !src)
520
		return 1;
521

522
	gather_stats(src, len, &stats);
523
	/* Optimization: No CRLF? Nothing to convert, regardless. */
524
	convert_crlf_into_lf = !!stats.crlf;
525

526
	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
527
		if (convert_is_binary(&stats))
528
			return 0;
529
		/*
530
		 * If the file in the index has any CR in it, do not
531
		 * convert.  This is the new safer autocrlf handling,
532
		 * unless we want to renormalize in a merge or
533
		 * cherry-pick.
534
		 */
535
		if ((!(conv_flags & CONV_EOL_RENORMALIZE)) &&
536
		    has_crlf_in_index(istate, path))
537
			convert_crlf_into_lf = 0;
538
	}
539
	if (((conv_flags & CONV_EOL_RNDTRP_WARN) ||
540
	     ((conv_flags & CONV_EOL_RNDTRP_DIE) && len))) {
541
		struct text_stat new_stats;
542
		memcpy(&new_stats, &stats, sizeof(new_stats));
543
		/* simulate "git add" */
544
		if (convert_crlf_into_lf) {
545
			new_stats.lonelf += new_stats.crlf;
546
			new_stats.crlf = 0;
547
		}
548
		/* simulate "git checkout" */
549
		if (will_convert_lf_to_crlf(&new_stats, crlf_action)) {
550
			new_stats.crlf += new_stats.lonelf;
551
			new_stats.lonelf = 0;
552
		}
553
		check_global_conv_flags_eol(path, &stats, &new_stats, conv_flags);
554
	}
555
	if (!convert_crlf_into_lf)
556
		return 0;
557

558
	/*
559
	 * At this point all of our source analysis is done, and we are sure we
560
	 * would convert. If we are in dry-run mode, we can give an answer.
561
	 */
562
	if (!buf)
563
		return 1;
564

565
	/* only grow if not in place */
566
	if (strbuf_avail(buf) + buf->len < len)
567
		strbuf_grow(buf, len - buf->len);
568
	dst = buf->buf;
569
	if (crlf_action == CRLF_AUTO || crlf_action == CRLF_AUTO_INPUT || crlf_action == CRLF_AUTO_CRLF) {
570
		/*
571
		 * If we guessed, we already know we rejected a file with
572
		 * lone CR, and we can strip a CR without looking at what
573
		 * follow it.
574
		 */
575
		do {
576
			unsigned char c = *src++;
577
			if (c != '\r')
578
				*dst++ = c;
579
		} while (--len);
580
	} else {
581
		do {
582
			unsigned char c = *src++;
583
			if (! (c == '\r' && (1 < len && *src == '\n')))
584
				*dst++ = c;
585
		} while (--len);
586
	}
587
	strbuf_setlen(buf, dst - buf->buf);
588
	return 1;
589
}
590

591
static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf,
592
			    enum convert_crlf_action crlf_action)
593
{
594
	char *to_free = NULL;
595
	struct text_stat stats;
596

597
	if (!len || output_eol(crlf_action) != EOL_CRLF)
598
		return 0;
599

600
	gather_stats(src, len, &stats);
601
	if (!will_convert_lf_to_crlf(&stats, crlf_action))
602
		return 0;
603

604
	/* are we "faking" in place editing ? */
605
	if (src == buf->buf)
606
		to_free = strbuf_detach(buf, NULL);
607

608
	strbuf_grow(buf, len + stats.lonelf);
609
	for (;;) {
610
		const char *nl = memchr(src, '\n', len);
611
		if (!nl)
612
			break;
613
		if (nl > src && nl[-1] == '\r') {
614
			strbuf_add(buf, src, nl + 1 - src);
615
		} else {
616
			strbuf_add(buf, src, nl - src);
617
			strbuf_addstr(buf, "\r\n");
618
		}
619
		len -= nl + 1 - src;
620
		src  = nl + 1;
621
	}
622
	strbuf_add(buf, src, len);
623

624
	free(to_free);
625
	return 1;
626
}
627

628
struct filter_params {
629
	const char *src;
630
	size_t size;
631
	int fd;
632
	const char *cmd;
633
	const char *path;
634
};
635

636
static int filter_buffer_or_fd(int in UNUSED, int out, void *data)
637
{
638
	/*
639
	 * Spawn cmd and feed the buffer contents through its stdin.
640
	 */
641
	struct child_process child_process = CHILD_PROCESS_INIT;
642
	struct filter_params *params = (struct filter_params *)data;
643
	const char *format = params->cmd;
644
	int write_err, status;
645

646
	/* apply % substitution to cmd */
647
	struct strbuf cmd = STRBUF_INIT;
648

649
	/* expand all %f with the quoted path; quote to preserve space, etc. */
650
	while (strbuf_expand_step(&cmd, &format)) {
651
		if (skip_prefix(format, "%", &format))
652
			strbuf_addch(&cmd, '%');
653
		else if (skip_prefix(format, "f", &format))
654
			sq_quote_buf(&cmd, params->path);
655
		else
656
			strbuf_addch(&cmd, '%');
657
	}
658

659
	strvec_push(&child_process.args, cmd.buf);
660
	child_process.use_shell = 1;
661
	child_process.in = -1;
662
	child_process.out = out;
663

664
	if (start_command(&child_process)) {
665
		strbuf_release(&cmd);
666
		return error(_("cannot fork to run external filter '%s'"),
667
			     params->cmd);
668
	}
669

670
	sigchain_push(SIGPIPE, SIG_IGN);
671

672
	if (params->src) {
673
		write_err = (write_in_full(child_process.in,
674
					   params->src, params->size) < 0);
675
		if (errno == EPIPE)
676
			write_err = 0;
677
	} else {
678
		write_err = copy_fd(params->fd, child_process.in);
679
		if (write_err == COPY_WRITE_ERROR && errno == EPIPE)
680
			write_err = 0;
681
	}
682

683
	if (close(child_process.in))
684
		write_err = 1;
685
	if (write_err)
686
		error(_("cannot feed the input to external filter '%s'"),
687
		      params->cmd);
688

689
	sigchain_pop(SIGPIPE);
690

691
	status = finish_command(&child_process);
692
	if (status)
693
		error(_("external filter '%s' failed %d"), params->cmd, status);
694

695
	strbuf_release(&cmd);
696
	return (write_err || status);
697
}
698

699
static int apply_single_file_filter(const char *path, const char *src, size_t len, int fd,
700
				    struct strbuf *dst, const char *cmd)
701
{
702
	/*
703
	 * Create a pipeline to have the command filter the buffer's
704
	 * contents.
705
	 *
706
	 * (child --> cmd) --> us
707
	 */
708
	int err = 0;
709
	struct strbuf nbuf = STRBUF_INIT;
710
	struct async async;
711
	struct filter_params params;
712

713
	memset(&async, 0, sizeof(async));
714
	async.proc = filter_buffer_or_fd;
715
	async.data = &params;
716
	async.out = -1;
717
	params.src = src;
718
	params.size = len;
719
	params.fd = fd;
720
	params.cmd = cmd;
721
	params.path = path;
722

723
	fflush(NULL);
724
	if (start_async(&async))
725
		return 0;	/* error was already reported */
726

727
	if (strbuf_read(&nbuf, async.out, 0) < 0) {
728
		err = error(_("read from external filter '%s' failed"), cmd);
729
	}
730
	if (close(async.out)) {
731
		err = error(_("read from external filter '%s' failed"), cmd);
732
	}
733
	if (finish_async(&async)) {
734
		err = error(_("external filter '%s' failed"), cmd);
735
	}
736

737
	if (!err) {
738
		strbuf_swap(dst, &nbuf);
739
	}
740
	strbuf_release(&nbuf);
741
	return !err;
742
}
743

744
#define CAP_CLEAN    (1u<<0)
745
#define CAP_SMUDGE   (1u<<1)
746
#define CAP_DELAY    (1u<<2)
747

748
struct cmd2process {
749
	struct subprocess_entry subprocess; /* must be the first member! */
750
	unsigned int supported_capabilities;
751
};
752

753
static int subprocess_map_initialized;
754
static struct hashmap subprocess_map;
755

756
static int start_multi_file_filter_fn(struct subprocess_entry *subprocess)
757
{
758
	static int versions[] = {2, 0};
759
	static struct subprocess_capability capabilities[] = {
760
		{ "clean",  CAP_CLEAN  },
761
		{ "smudge", CAP_SMUDGE },
762
		{ "delay",  CAP_DELAY  },
763
		{ NULL, 0 }
764
	};
765
	struct cmd2process *entry = (struct cmd2process *)subprocess;
766
	return subprocess_handshake(subprocess, "git-filter", versions, NULL,
767
				    capabilities,
768
				    &entry->supported_capabilities);
769
}
770

771
static void handle_filter_error(const struct strbuf *filter_status,
772
				struct cmd2process *entry,
773
				const unsigned int wanted_capability)
774
{
775
	if (!strcmp(filter_status->buf, "error"))
776
		; /* The filter signaled a problem with the file. */
777
	else if (!strcmp(filter_status->buf, "abort") && wanted_capability) {
778
		/*
779
		 * The filter signaled a permanent problem. Don't try to filter
780
		 * files with the same command for the lifetime of the current
781
		 * Git process.
782
		 */
783
		 entry->supported_capabilities &= ~wanted_capability;
784
	} else {
785
		/*
786
		 * Something went wrong with the protocol filter.
787
		 * Force shutdown and restart if another blob requires filtering.
788
		 */
789
		error(_("external filter '%s' failed"), entry->subprocess.cmd);
790
		subprocess_stop(&subprocess_map, &entry->subprocess);
791
		free(entry);
792
	}
793
}
794

795
static int apply_multi_file_filter(const char *path, const char *src, size_t len,
796
				   int fd, struct strbuf *dst, const char *cmd,
797
				   const unsigned int wanted_capability,
798
				   const struct checkout_metadata *meta,
799
				   struct delayed_checkout *dco)
800
{
801
	int err;
802
	int can_delay = 0;
803
	struct cmd2process *entry;
804
	struct child_process *process;
805
	struct strbuf nbuf = STRBUF_INIT;
806
	struct strbuf filter_status = STRBUF_INIT;
807
	const char *filter_type;
808

809
	if (!subprocess_map_initialized) {
810
		subprocess_map_initialized = 1;
811
		hashmap_init(&subprocess_map, cmd2process_cmp, NULL, 0);
812
		entry = NULL;
813
	} else {
814
		entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
815
	}
816

817
	fflush(NULL);
818

819
	if (!entry) {
820
		entry = xmalloc(sizeof(*entry));
821
		entry->supported_capabilities = 0;
822

823
		if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_multi_file_filter_fn)) {
824
			free(entry);
825
			return 0;
826
		}
827
	}
828
	process = &entry->subprocess.process;
829

830
	if (!(entry->supported_capabilities & wanted_capability))
831
		return 0;
832

833
	if (wanted_capability & CAP_CLEAN)
834
		filter_type = "clean";
835
	else if (wanted_capability & CAP_SMUDGE)
836
		filter_type = "smudge";
837
	else
838
		die(_("unexpected filter type"));
839

840
	sigchain_push(SIGPIPE, SIG_IGN);
841

842
	assert(strlen(filter_type) < LARGE_PACKET_DATA_MAX - strlen("command=\n"));
843
	err = packet_write_fmt_gently(process->in, "command=%s\n", filter_type);
844
	if (err)
845
		goto done;
846

847
	err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
848
	if (err) {
849
		error(_("path name too long for external filter"));
850
		goto done;
851
	}
852

853
	err = packet_write_fmt_gently(process->in, "pathname=%s\n", path);
854
	if (err)
855
		goto done;
856

857
	if (meta && meta->refname) {
858
		err = packet_write_fmt_gently(process->in, "ref=%s\n", meta->refname);
859
		if (err)
860
			goto done;
861
	}
862

863
	if (meta && !is_null_oid(&meta->treeish)) {
864
		err = packet_write_fmt_gently(process->in, "treeish=%s\n", oid_to_hex(&meta->treeish));
865
		if (err)
866
			goto done;
867
	}
868

869
	if (meta && !is_null_oid(&meta->blob)) {
870
		err = packet_write_fmt_gently(process->in, "blob=%s\n", oid_to_hex(&meta->blob));
871
		if (err)
872
			goto done;
873
	}
874

875
	if ((entry->supported_capabilities & CAP_DELAY) &&
876
	    dco && dco->state == CE_CAN_DELAY) {
877
		can_delay = 1;
878
		err = packet_write_fmt_gently(process->in, "can-delay=1\n");
879
		if (err)
880
			goto done;
881
	}
882

883
	err = packet_flush_gently(process->in);
884
	if (err)
885
		goto done;
886

887
	if (fd >= 0)
888
		err = write_packetized_from_fd_no_flush(fd, process->in);
889
	else
890
		err = write_packetized_from_buf_no_flush(src, len, process->in);
891
	if (err)
892
		goto done;
893

894
	err = packet_flush_gently(process->in);
895
	if (err)
896
		goto done;
897

898
	err = subprocess_read_status(process->out, &filter_status);
899
	if (err)
900
		goto done;
901

902
	if (can_delay && !strcmp(filter_status.buf, "delayed")) {
903
		string_list_insert(&dco->filters, cmd);
904
		string_list_insert(&dco->paths, path);
905
	} else {
906
		/* The filter got the blob and wants to send us a response. */
907
		err = strcmp(filter_status.buf, "success");
908
		if (err)
909
			goto done;
910

911
		err = read_packetized_to_strbuf(process->out, &nbuf,
912
						PACKET_READ_GENTLE_ON_EOF) < 0;
913
		if (err)
914
			goto done;
915

916
		err = subprocess_read_status(process->out, &filter_status);
917
		if (err)
918
			goto done;
919

920
		err = strcmp(filter_status.buf, "success");
921
	}
922

923
done:
924
	sigchain_pop(SIGPIPE);
925

926
	if (err)
927
		handle_filter_error(&filter_status, entry, wanted_capability);
928
	else
929
		strbuf_swap(dst, &nbuf);
930
	strbuf_release(&nbuf);
931
	strbuf_release(&filter_status);
932
	return !err;
933
}
934

935

936
int async_query_available_blobs(const char *cmd, struct string_list *available_paths)
937
{
938
	int err;
939
	char *line;
940
	struct cmd2process *entry;
941
	struct child_process *process;
942
	struct strbuf filter_status = STRBUF_INIT;
943

944
	assert(subprocess_map_initialized);
945
	entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
946
	if (!entry) {
947
		error(_("external filter '%s' is not available anymore although "
948
			"not all paths have been filtered"), cmd);
949
		return 0;
950
	}
951
	process = &entry->subprocess.process;
952
	sigchain_push(SIGPIPE, SIG_IGN);
953

954
	err = packet_write_fmt_gently(
955
		process->in, "command=list_available_blobs\n");
956
	if (err)
957
		goto done;
958

959
	err = packet_flush_gently(process->in);
960
	if (err)
961
		goto done;
962

963
	while ((line = packet_read_line(process->out, NULL))) {
964
		const char *path;
965
		if (skip_prefix(line, "pathname=", &path))
966
			string_list_insert(available_paths, path);
967
		else
968
			; /* ignore unknown keys */
969
	}
970

971
	err = subprocess_read_status(process->out, &filter_status);
972
	if (err)
973
		goto done;
974

975
	err = strcmp(filter_status.buf, "success");
976

977
done:
978
	sigchain_pop(SIGPIPE);
979

980
	if (err)
981
		handle_filter_error(&filter_status, entry, 0);
982
	strbuf_release(&filter_status);
983
	return !err;
984
}
985

986
static struct convert_driver {
987
	const char *name;
988
	struct convert_driver *next;
989
	char *smudge;
990
	char *clean;
991
	char *process;
992
	int required;
993
} *user_convert, **user_convert_tail;
994

995
static int apply_filter(const char *path, const char *src, size_t len,
996
			int fd, struct strbuf *dst, struct convert_driver *drv,
997
			const unsigned int wanted_capability,
998
			const struct checkout_metadata *meta,
999
			struct delayed_checkout *dco)
1000
{
1001
	const char *cmd = NULL;
1002

1003
	if (!drv)
1004
		return 0;
1005

1006
	if (!dst)
1007
		return 1;
1008

1009
	if ((wanted_capability & CAP_CLEAN) && !drv->process && drv->clean)
1010
		cmd = drv->clean;
1011
	else if ((wanted_capability & CAP_SMUDGE) && !drv->process && drv->smudge)
1012
		cmd = drv->smudge;
1013

1014
	if (cmd && *cmd)
1015
		return apply_single_file_filter(path, src, len, fd, dst, cmd);
1016
	else if (drv->process && *drv->process)
1017
		return apply_multi_file_filter(path, src, len, fd, dst,
1018
			drv->process, wanted_capability, meta, dco);
1019

1020
	return 0;
1021
}
1022

1023
static int read_convert_config(const char *var, const char *value,
1024
			       const struct config_context *ctx UNUSED,
1025
			       void *cb UNUSED)
1026
{
1027
	const char *key, *name;
1028
	size_t namelen;
1029
	struct convert_driver *drv;
1030

1031
	/*
1032
	 * External conversion drivers are configured using
1033
	 * "filter.<name>.variable".
1034
	 */
1035
	if (parse_config_key(var, "filter", &name, &namelen, &key) < 0 || !name)
1036
		return 0;
1037
	for (drv = user_convert; drv; drv = drv->next)
1038
		if (!xstrncmpz(drv->name, name, namelen))
1039
			break;
1040
	if (!drv) {
1041
		CALLOC_ARRAY(drv, 1);
1042
		drv->name = xmemdupz(name, namelen);
1043
		*user_convert_tail = drv;
1044
		user_convert_tail = &(drv->next);
1045
	}
1046

1047
	/*
1048
	 * filter.<name>.smudge and filter.<name>.clean specifies
1049
	 * the command line:
1050
	 *
1051
	 *	command-line
1052
	 *
1053
	 * The command-line will not be interpolated in any way.
1054
	 */
1055

1056
	if (!strcmp("smudge", key)) {
1057
		FREE_AND_NULL(drv->smudge);
1058
		return git_config_string(&drv->smudge, var, value);
1059
	}
1060

1061
	if (!strcmp("clean", key)) {
1062
		FREE_AND_NULL(drv->clean);
1063
		return git_config_string(&drv->clean, var, value);
1064
	}
1065

1066
	if (!strcmp("process", key)) {
1067
		FREE_AND_NULL(drv->process);
1068
		return git_config_string(&drv->process, var, value);
1069
	}
1070

1071
	if (!strcmp("required", key)) {
1072
		drv->required = git_config_bool(var, value);
1073
		return 0;
1074
	}
1075

1076
	return 0;
1077
}
1078

1079
static int count_ident(const char *cp, unsigned long size)
1080
{
1081
	/*
1082
	 * "$Id: 0000000000000000000000000000000000000000 $" <=> "$Id$"
1083
	 */
1084
	int cnt = 0;
1085
	char ch;
1086

1087
	while (size) {
1088
		ch = *cp++;
1089
		size--;
1090
		if (ch != '$')
1091
			continue;
1092
		if (size < 3)
1093
			break;
1094
		if (memcmp("Id", cp, 2))
1095
			continue;
1096
		ch = cp[2];
1097
		cp += 3;
1098
		size -= 3;
1099
		if (ch == '$')
1100
			cnt++; /* $Id$ */
1101
		if (ch != ':')
1102
			continue;
1103

1104
		/*
1105
		 * "$Id: ... "; scan up to the closing dollar sign and discard.
1106
		 */
1107
		while (size) {
1108
			ch = *cp++;
1109
			size--;
1110
			if (ch == '$') {
1111
				cnt++;
1112
				break;
1113
			}
1114
			if (ch == '\n')
1115
				break;
1116
		}
1117
	}
1118
	return cnt;
1119
}
1120

1121
static int ident_to_git(const char *src, size_t len,
1122
			struct strbuf *buf, int ident)
1123
{
1124
	char *dst, *dollar;
1125

1126
	if (!ident || (src && !count_ident(src, len)))
1127
		return 0;
1128

1129
	if (!buf)
1130
		return 1;
1131

1132
	/* only grow if not in place */
1133
	if (strbuf_avail(buf) + buf->len < len)
1134
		strbuf_grow(buf, len - buf->len);
1135
	dst = buf->buf;
1136
	for (;;) {
1137
		dollar = memchr(src, '$', len);
1138
		if (!dollar)
1139
			break;
1140
		memmove(dst, src, dollar + 1 - src);
1141
		dst += dollar + 1 - src;
1142
		len -= dollar + 1 - src;
1143
		src  = dollar + 1;
1144

1145
		if (len > 3 && !memcmp(src, "Id:", 3)) {
1146
			dollar = memchr(src + 3, '$', len - 3);
1147
			if (!dollar)
1148
				break;
1149
			if (memchr(src + 3, '\n', dollar - src - 3)) {
1150
				/* Line break before the next dollar. */
1151
				continue;
1152
			}
1153

1154
			memcpy(dst, "Id$", 3);
1155
			dst += 3;
1156
			len -= dollar + 1 - src;
1157
			src  = dollar + 1;
1158
		}
1159
	}
1160
	memmove(dst, src, len);
1161
	strbuf_setlen(buf, dst + len - buf->buf);
1162
	return 1;
1163
}
1164

1165
static int ident_to_worktree(const char *src, size_t len,
1166
			     struct strbuf *buf, int ident)
1167
{
1168
	struct object_id oid;
1169
	char *to_free = NULL, *dollar, *spc;
1170
	int cnt;
1171

1172
	if (!ident)
1173
		return 0;
1174

1175
	cnt = count_ident(src, len);
1176
	if (!cnt)
1177
		return 0;
1178

1179
	/* are we "faking" in place editing ? */
1180
	if (src == buf->buf)
1181
		to_free = strbuf_detach(buf, NULL);
1182
	hash_object_file(the_hash_algo, src, len, OBJ_BLOB, &oid);
1183

1184
	strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3));
1185
	for (;;) {
1186
		/* step 1: run to the next '$' */
1187
		dollar = memchr(src, '$', len);
1188
		if (!dollar)
1189
			break;
1190
		strbuf_add(buf, src, dollar + 1 - src);
1191
		len -= dollar + 1 - src;
1192
		src  = dollar + 1;
1193

1194
		/* step 2: does it looks like a bit like Id:xxx$ or Id$ ? */
1195
		if (len < 3 || memcmp("Id", src, 2))
1196
			continue;
1197

1198
		/* step 3: skip over Id$ or Id:xxxxx$ */
1199
		if (src[2] == '$') {
1200
			src += 3;
1201
			len -= 3;
1202
		} else if (src[2] == ':') {
1203
			/*
1204
			 * It's possible that an expanded Id has crept its way into the
1205
			 * repository, we cope with that by stripping the expansion out.
1206
			 * This is probably not a good idea, since it will cause changes
1207
			 * on checkout, which won't go away by stash, but let's keep it
1208
			 * for git-style ids.
1209
			 */
1210
			dollar = memchr(src + 3, '$', len - 3);
1211
			if (!dollar) {
1212
				/* incomplete keyword, no more '$', so just quit the loop */
1213
				break;
1214
			}
1215

1216
			if (memchr(src + 3, '\n', dollar - src - 3)) {
1217
				/* Line break before the next dollar. */
1218
				continue;
1219
			}
1220

1221
			spc = memchr(src + 4, ' ', dollar - src - 4);
1222
			if (spc && spc < dollar-1) {
1223
				/* There are spaces in unexpected places.
1224
				 * This is probably an id from some other
1225
				 * versioning system. Keep it for now.
1226
				 */
1227
				continue;
1228
			}
1229

1230
			len -= dollar + 1 - src;
1231
			src  = dollar + 1;
1232
		} else {
1233
			/* it wasn't a "Id$" or "Id:xxxx$" */
1234
			continue;
1235
		}
1236

1237
		/* step 4: substitute */
1238
		strbuf_addstr(buf, "Id: ");
1239
		strbuf_addstr(buf, oid_to_hex(&oid));
1240
		strbuf_addstr(buf, " $");
1241
	}
1242
	strbuf_add(buf, src, len);
1243

1244
	free(to_free);
1245
	return 1;
1246
}
1247

1248
static const char *git_path_check_encoding(struct attr_check_item *check)
1249
{
1250
	const char *value = check->value;
1251

1252
	if (ATTR_UNSET(value) || !strlen(value))
1253
		return NULL;
1254

1255
	if (ATTR_TRUE(value) || ATTR_FALSE(value)) {
1256
		die(_("true/false are no valid working-tree-encodings"));
1257
	}
1258

1259
	/* Don't encode to the default encoding */
1260
	if (same_encoding(value, default_encoding))
1261
		return NULL;
1262

1263
	return value;
1264
}
1265

1266
static enum convert_crlf_action git_path_check_crlf(struct attr_check_item *check)
1267
{
1268
	const char *value = check->value;
1269

1270
	if (ATTR_TRUE(value))
1271
		return CRLF_TEXT;
1272
	else if (ATTR_FALSE(value))
1273
		return CRLF_BINARY;
1274
	else if (ATTR_UNSET(value))
1275
		;
1276
	else if (!strcmp(value, "input"))
1277
		return CRLF_TEXT_INPUT;
1278
	else if (!strcmp(value, "auto"))
1279
		return CRLF_AUTO;
1280
	return CRLF_UNDEFINED;
1281
}
1282

1283
static enum eol git_path_check_eol(struct attr_check_item *check)
1284
{
1285
	const char *value = check->value;
1286

1287
	if (ATTR_UNSET(value))
1288
		;
1289
	else if (!strcmp(value, "lf"))
1290
		return EOL_LF;
1291
	else if (!strcmp(value, "crlf"))
1292
		return EOL_CRLF;
1293
	return EOL_UNSET;
1294
}
1295

1296
static struct convert_driver *git_path_check_convert(struct attr_check_item *check)
1297
{
1298
	const char *value = check->value;
1299
	struct convert_driver *drv;
1300

1301
	if (ATTR_TRUE(value) || ATTR_FALSE(value) || ATTR_UNSET(value))
1302
		return NULL;
1303
	for (drv = user_convert; drv; drv = drv->next)
1304
		if (!strcmp(value, drv->name))
1305
			return drv;
1306
	return NULL;
1307
}
1308

1309
static int git_path_check_ident(struct attr_check_item *check)
1310
{
1311
	const char *value = check->value;
1312

1313
	return !!ATTR_TRUE(value);
1314
}
1315

1316
static struct attr_check *check;
1317

1318
void convert_attrs(struct index_state *istate,
1319
		   struct conv_attrs *ca, const char *path)
1320
{
1321
	struct attr_check_item *ccheck = NULL;
1322

1323
	if (!check) {
1324
		check = attr_check_initl("crlf", "ident", "filter",
1325
					 "eol", "text", "working-tree-encoding",
1326
					 NULL);
1327
		user_convert_tail = &user_convert;
1328
		git_config(read_convert_config, NULL);
1329
	}
1330

1331
	git_check_attr(istate, path, check);
1332
	ccheck = check->items;
1333
	ca->crlf_action = git_path_check_crlf(ccheck + 4);
1334
	if (ca->crlf_action == CRLF_UNDEFINED)
1335
		ca->crlf_action = git_path_check_crlf(ccheck + 0);
1336
	ca->ident = git_path_check_ident(ccheck + 1);
1337
	ca->drv = git_path_check_convert(ccheck + 2);
1338
	if (ca->crlf_action != CRLF_BINARY) {
1339
		enum eol eol_attr = git_path_check_eol(ccheck + 3);
1340
		if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_LF)
1341
			ca->crlf_action = CRLF_AUTO_INPUT;
1342
		else if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_CRLF)
1343
			ca->crlf_action = CRLF_AUTO_CRLF;
1344
		else if (eol_attr == EOL_LF)
1345
			ca->crlf_action = CRLF_TEXT_INPUT;
1346
		else if (eol_attr == EOL_CRLF)
1347
			ca->crlf_action = CRLF_TEXT_CRLF;
1348
	}
1349
	ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
1350

1351
	/* Save attr and make a decision for action */
1352
	ca->attr_action = ca->crlf_action;
1353
	if (ca->crlf_action == CRLF_TEXT)
1354
		ca->crlf_action = text_eol_is_crlf() ? CRLF_TEXT_CRLF : CRLF_TEXT_INPUT;
1355
	if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_FALSE)
1356
		ca->crlf_action = CRLF_BINARY;
1357
	if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_TRUE)
1358
		ca->crlf_action = CRLF_AUTO_CRLF;
1359
	if (ca->crlf_action == CRLF_UNDEFINED && auto_crlf == AUTO_CRLF_INPUT)
1360
		ca->crlf_action = CRLF_AUTO_INPUT;
1361
}
1362

1363
void reset_parsed_attributes(void)
1364
{
1365
	struct convert_driver *drv, *next;
1366

1367
	attr_check_free(check);
1368
	check = NULL;
1369
	reset_merge_attributes();
1370

1371
	for (drv = user_convert; drv; drv = next) {
1372
		next = drv->next;
1373
		free((void *)drv->name);
1374
		free(drv);
1375
	}
1376
	user_convert = NULL;
1377
	user_convert_tail = NULL;
1378
}
1379

1380
int would_convert_to_git_filter_fd(struct index_state *istate, const char *path)
1381
{
1382
	struct conv_attrs ca;
1383

1384
	convert_attrs(istate, &ca, path);
1385
	if (!ca.drv)
1386
		return 0;
1387

1388
	/*
1389
	 * Apply a filter to an fd only if the filter is required to succeed.
1390
	 * We must die if the filter fails, because the original data before
1391
	 * filtering is not available.
1392
	 */
1393
	if (!ca.drv->required)
1394
		return 0;
1395

1396
	return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL, NULL);
1397
}
1398

1399
const char *get_convert_attr_ascii(struct index_state *istate, const char *path)
1400
{
1401
	struct conv_attrs ca;
1402

1403
	convert_attrs(istate, &ca, path);
1404
	switch (ca.attr_action) {
1405
	case CRLF_UNDEFINED:
1406
		return "";
1407
	case CRLF_BINARY:
1408
		return "-text";
1409
	case CRLF_TEXT:
1410
		return "text";
1411
	case CRLF_TEXT_INPUT:
1412
		return "text eol=lf";
1413
	case CRLF_TEXT_CRLF:
1414
		return "text eol=crlf";
1415
	case CRLF_AUTO:
1416
		return "text=auto";
1417
	case CRLF_AUTO_CRLF:
1418
		return "text=auto eol=crlf";
1419
	case CRLF_AUTO_INPUT:
1420
		return "text=auto eol=lf";
1421
	}
1422
	return "";
1423
}
1424

1425
int convert_to_git(struct index_state *istate,
1426
		   const char *path, const char *src, size_t len,
1427
		   struct strbuf *dst, int conv_flags)
1428
{
1429
	int ret = 0;
1430
	struct conv_attrs ca;
1431

1432
	convert_attrs(istate, &ca, path);
1433

1434
	ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN, NULL, NULL);
1435
	if (!ret && ca.drv && ca.drv->required)
1436
		die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1437

1438
	if (ret && dst) {
1439
		src = dst->buf;
1440
		len = dst->len;
1441
	}
1442

1443
	ret |= encode_to_git(path, src, len, dst, ca.working_tree_encoding, conv_flags);
1444
	if (ret && dst) {
1445
		src = dst->buf;
1446
		len = dst->len;
1447
	}
1448

1449
	if (!(conv_flags & CONV_EOL_KEEP_CRLF)) {
1450
		ret |= crlf_to_git(istate, path, src, len, dst, ca.crlf_action, conv_flags);
1451
		if (ret && dst) {
1452
			src = dst->buf;
1453
			len = dst->len;
1454
		}
1455
	}
1456
	return ret | ident_to_git(src, len, dst, ca.ident);
1457
}
1458

1459
void convert_to_git_filter_fd(struct index_state *istate,
1460
			      const char *path, int fd, struct strbuf *dst,
1461
			      int conv_flags)
1462
{
1463
	struct conv_attrs ca;
1464
	convert_attrs(istate, &ca, path);
1465

1466
	assert(ca.drv);
1467

1468
	if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL, NULL))
1469
		die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
1470

1471
	encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags);
1472
	crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags);
1473
	ident_to_git(dst->buf, dst->len, dst, ca.ident);
1474
}
1475

1476
static int convert_to_working_tree_ca_internal(const struct conv_attrs *ca,
1477
					       const char *path, const char *src,
1478
					       size_t len, struct strbuf *dst,
1479
					       int normalizing,
1480
					       const struct checkout_metadata *meta,
1481
					       struct delayed_checkout *dco)
1482
{
1483
	int ret = 0, ret_filter = 0;
1484

1485
	ret |= ident_to_worktree(src, len, dst, ca->ident);
1486
	if (ret) {
1487
		src = dst->buf;
1488
		len = dst->len;
1489
	}
1490
	/*
1491
	 * CRLF conversion can be skipped if normalizing, unless there
1492
	 * is a smudge or process filter (even if the process filter doesn't
1493
	 * support smudge).  The filters might expect CRLFs.
1494
	 */
1495
	if ((ca->drv && (ca->drv->smudge || ca->drv->process)) || !normalizing) {
1496
		ret |= crlf_to_worktree(src, len, dst, ca->crlf_action);
1497
		if (ret) {
1498
			src = dst->buf;
1499
			len = dst->len;
1500
		}
1501
	}
1502

1503
	ret |= encode_to_worktree(path, src, len, dst, ca->working_tree_encoding);
1504
	if (ret) {
1505
		src = dst->buf;
1506
		len = dst->len;
1507
	}
1508

1509
	ret_filter = apply_filter(
1510
		path, src, len, -1, dst, ca->drv, CAP_SMUDGE, meta, dco);
1511
	if (!ret_filter && ca->drv && ca->drv->required)
1512
		die(_("%s: smudge filter %s failed"), path, ca->drv->name);
1513

1514
	return ret | ret_filter;
1515
}
1516

1517
int async_convert_to_working_tree_ca(const struct conv_attrs *ca,
1518
				     const char *path, const char *src,
1519
				     size_t len, struct strbuf *dst,
1520
				     const struct checkout_metadata *meta,
1521
				     void *dco)
1522
{
1523
	return convert_to_working_tree_ca_internal(ca, path, src, len, dst, 0,
1524
						   meta, dco);
1525
}
1526

1527
int convert_to_working_tree_ca(const struct conv_attrs *ca,
1528
			       const char *path, const char *src,
1529
			       size_t len, struct strbuf *dst,
1530
			       const struct checkout_metadata *meta)
1531
{
1532
	return convert_to_working_tree_ca_internal(ca, path, src, len, dst, 0,
1533
						   meta, NULL);
1534
}
1535

1536
int renormalize_buffer(struct index_state *istate, const char *path,
1537
		       const char *src, size_t len, struct strbuf *dst)
1538
{
1539
	struct conv_attrs ca;
1540
	int ret;
1541

1542
	convert_attrs(istate, &ca, path);
1543
	ret = convert_to_working_tree_ca_internal(&ca, path, src, len, dst, 1,
1544
						  NULL, NULL);
1545
	if (ret) {
1546
		src = dst->buf;
1547
		len = dst->len;
1548
	}
1549
	return ret | convert_to_git(istate, path, src, len, dst, CONV_EOL_RENORMALIZE);
1550
}
1551

1552
/*****************************************************************
1553
 *
1554
 * Streaming conversion support
1555
 *
1556
 *****************************************************************/
1557

1558
typedef int (*filter_fn)(struct stream_filter *,
1559
			 const char *input, size_t *isize_p,
1560
			 char *output, size_t *osize_p);
1561
typedef void (*free_fn)(struct stream_filter *);
1562

1563
struct stream_filter_vtbl {
1564
	filter_fn filter;
1565
	free_fn free;
1566
};
1567

1568
struct stream_filter {
1569
	struct stream_filter_vtbl *vtbl;
1570
};
1571

1572
static int null_filter_fn(struct stream_filter *filter UNUSED,
1573
			  const char *input, size_t *isize_p,
1574
			  char *output, size_t *osize_p)
1575
{
1576
	size_t count;
1577

1578
	if (!input)
1579
		return 0; /* we do not keep any states */
1580
	count = *isize_p;
1581
	if (*osize_p < count)
1582
		count = *osize_p;
1583
	if (count) {
1584
		memmove(output, input, count);
1585
		*isize_p -= count;
1586
		*osize_p -= count;
1587
	}
1588
	return 0;
1589
}
1590

1591
static void null_free_fn(struct stream_filter *filter UNUSED)
1592
{
1593
	; /* nothing -- null instances are shared */
1594
}
1595

1596
static struct stream_filter_vtbl null_vtbl = {
1597
	.filter = null_filter_fn,
1598
	.free = null_free_fn,
1599
};
1600

1601
static struct stream_filter null_filter_singleton = {
1602
	.vtbl = &null_vtbl,
1603
};
1604

1605
int is_null_stream_filter(struct stream_filter *filter)
1606
{
1607
	return filter == &null_filter_singleton;
1608
}
1609

1610

1611
/*
1612
 * LF-to-CRLF filter
1613
 */
1614

1615
struct lf_to_crlf_filter {
1616
	struct stream_filter filter;
1617
	unsigned has_held:1;
1618
	char held;
1619
};
1620

1621
static int lf_to_crlf_filter_fn(struct stream_filter *filter,
1622
				const char *input, size_t *isize_p,
1623
				char *output, size_t *osize_p)
1624
{
1625
	size_t count, o = 0;
1626
	struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter;
1627

1628
	/*
1629
	 * We may be holding onto the CR to see if it is followed by a
1630
	 * LF, in which case we would need to go to the main loop.
1631
	 * Otherwise, just emit it to the output stream.
1632
	 */
1633
	if (lf_to_crlf->has_held && (lf_to_crlf->held != '\r' || !input)) {
1634
		output[o++] = lf_to_crlf->held;
1635
		lf_to_crlf->has_held = 0;
1636
	}
1637

1638
	/* We are told to drain */
1639
	if (!input) {
1640
		*osize_p -= o;
1641
		return 0;
1642
	}
1643

1644
	count = *isize_p;
1645
	if (count || lf_to_crlf->has_held) {
1646
		size_t i;
1647
		int was_cr = 0;
1648

1649
		if (lf_to_crlf->has_held) {
1650
			was_cr = 1;
1651
			lf_to_crlf->has_held = 0;
1652
		}
1653

1654
		for (i = 0; o < *osize_p && i < count; i++) {
1655
			char ch = input[i];
1656

1657
			if (ch == '\n') {
1658
				output[o++] = '\r';
1659
			} else if (was_cr) {
1660
				/*
1661
				 * Previous round saw CR and it is not followed
1662
				 * by a LF; emit the CR before processing the
1663
				 * current character.
1664
				 */
1665
				output[o++] = '\r';
1666
			}
1667

1668
			/*
1669
			 * We may have consumed the last output slot,
1670
			 * in which case we need to break out of this
1671
			 * loop; hold the current character before
1672
			 * returning.
1673
			 */
1674
			if (*osize_p <= o) {
1675
				lf_to_crlf->has_held = 1;
1676
				lf_to_crlf->held = ch;
1677
				continue; /* break but increment i */
1678
			}
1679

1680
			if (ch == '\r') {
1681
				was_cr = 1;
1682
				continue;
1683
			}
1684

1685
			was_cr = 0;
1686
			output[o++] = ch;
1687
		}
1688

1689
		*osize_p -= o;
1690
		*isize_p -= i;
1691

1692
		if (!lf_to_crlf->has_held && was_cr) {
1693
			lf_to_crlf->has_held = 1;
1694
			lf_to_crlf->held = '\r';
1695
		}
1696
	}
1697
	return 0;
1698
}
1699

1700
static void lf_to_crlf_free_fn(struct stream_filter *filter)
1701
{
1702
	free(filter);
1703
}
1704

1705
static struct stream_filter_vtbl lf_to_crlf_vtbl = {
1706
	.filter = lf_to_crlf_filter_fn,
1707
	.free = lf_to_crlf_free_fn,
1708
};
1709

1710
static struct stream_filter *lf_to_crlf_filter(void)
1711
{
1712
	struct lf_to_crlf_filter *lf_to_crlf = xcalloc(1, sizeof(*lf_to_crlf));
1713

1714
	lf_to_crlf->filter.vtbl = &lf_to_crlf_vtbl;
1715
	return (struct stream_filter *)lf_to_crlf;
1716
}
1717

1718
/*
1719
 * Cascade filter
1720
 */
1721
#define FILTER_BUFFER 1024
1722
struct cascade_filter {
1723
	struct stream_filter filter;
1724
	struct stream_filter *one;
1725
	struct stream_filter *two;
1726
	char buf[FILTER_BUFFER];
1727
	int end, ptr;
1728
};
1729

1730
static int cascade_filter_fn(struct stream_filter *filter,
1731
			     const char *input, size_t *isize_p,
1732
			     char *output, size_t *osize_p)
1733
{
1734
	struct cascade_filter *cas = (struct cascade_filter *) filter;
1735
	size_t filled = 0;
1736
	size_t sz = *osize_p;
1737
	size_t to_feed, remaining;
1738

1739
	/*
1740
	 * input -- (one) --> buf -- (two) --> output
1741
	 */
1742
	while (filled < sz) {
1743
		remaining = sz - filled;
1744

1745
		/* do we already have something to feed two with? */
1746
		if (cas->ptr < cas->end) {
1747
			to_feed = cas->end - cas->ptr;
1748
			if (stream_filter(cas->two,
1749
					  cas->buf + cas->ptr, &to_feed,
1750
					  output + filled, &remaining))
1751
				return -1;
1752
			cas->ptr += (cas->end - cas->ptr) - to_feed;
1753
			filled = sz - remaining;
1754
			continue;
1755
		}
1756

1757
		/* feed one from upstream and have it emit into our buffer */
1758
		to_feed = input ? *isize_p : 0;
1759
		if (input && !to_feed)
1760
			break;
1761
		remaining = sizeof(cas->buf);
1762
		if (stream_filter(cas->one,
1763
				  input, &to_feed,
1764
				  cas->buf, &remaining))
1765
			return -1;
1766
		cas->end = sizeof(cas->buf) - remaining;
1767
		cas->ptr = 0;
1768
		if (input) {
1769
			size_t fed = *isize_p - to_feed;
1770
			*isize_p -= fed;
1771
			input += fed;
1772
		}
1773

1774
		/* do we know that we drained one completely? */
1775
		if (input || cas->end)
1776
			continue;
1777

1778
		/* tell two to drain; we have nothing more to give it */
1779
		to_feed = 0;
1780
		remaining = sz - filled;
1781
		if (stream_filter(cas->two,
1782
				  NULL, &to_feed,
1783
				  output + filled, &remaining))
1784
			return -1;
1785
		if (remaining == (sz - filled))
1786
			break; /* completely drained two */
1787
		filled = sz - remaining;
1788
	}
1789
	*osize_p -= filled;
1790
	return 0;
1791
}
1792

1793
static void cascade_free_fn(struct stream_filter *filter)
1794
{
1795
	struct cascade_filter *cas = (struct cascade_filter *)filter;
1796
	free_stream_filter(cas->one);
1797
	free_stream_filter(cas->two);
1798
	free(filter);
1799
}
1800

1801
static struct stream_filter_vtbl cascade_vtbl = {
1802
	.filter = cascade_filter_fn,
1803
	.free = cascade_free_fn,
1804
};
1805

1806
static struct stream_filter *cascade_filter(struct stream_filter *one,
1807
					    struct stream_filter *two)
1808
{
1809
	struct cascade_filter *cascade;
1810

1811
	if (!one || is_null_stream_filter(one))
1812
		return two;
1813
	if (!two || is_null_stream_filter(two))
1814
		return one;
1815

1816
	cascade = xmalloc(sizeof(*cascade));
1817
	cascade->one = one;
1818
	cascade->two = two;
1819
	cascade->end = cascade->ptr = 0;
1820
	cascade->filter.vtbl = &cascade_vtbl;
1821
	return (struct stream_filter *)cascade;
1822
}
1823

1824
/*
1825
 * ident filter
1826
 */
1827
#define IDENT_DRAINING (-1)
1828
#define IDENT_SKIPPING (-2)
1829
struct ident_filter {
1830
	struct stream_filter filter;
1831
	struct strbuf left;
1832
	int state;
1833
	char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */
1834
};
1835

1836
static int is_foreign_ident(const char *str)
1837
{
1838
	int i;
1839

1840
	if (!skip_prefix(str, "$Id: ", &str))
1841
		return 0;
1842
	for (i = 0; str[i]; i++) {
1843
		if (isspace(str[i]) && str[i+1] != '$')
1844
			return 1;
1845
	}
1846
	return 0;
1847
}
1848

1849
static void ident_drain(struct ident_filter *ident, char **output_p, size_t *osize_p)
1850
{
1851
	size_t to_drain = ident->left.len;
1852

1853
	if (*osize_p < to_drain)
1854
		to_drain = *osize_p;
1855
	if (to_drain) {
1856
		memcpy(*output_p, ident->left.buf, to_drain);
1857
		strbuf_remove(&ident->left, 0, to_drain);
1858
		*output_p += to_drain;
1859
		*osize_p -= to_drain;
1860
	}
1861
	if (!ident->left.len)
1862
		ident->state = 0;
1863
}
1864

1865
static int ident_filter_fn(struct stream_filter *filter,
1866
			   const char *input, size_t *isize_p,
1867
			   char *output, size_t *osize_p)
1868
{
1869
	struct ident_filter *ident = (struct ident_filter *)filter;
1870
	static const char head[] = "$Id";
1871

1872
	if (!input) {
1873
		/* drain upon eof */
1874
		switch (ident->state) {
1875
		default:
1876
			strbuf_add(&ident->left, head, ident->state);
1877
			/* fallthrough */
1878
		case IDENT_SKIPPING:
1879
			/* fallthrough */
1880
		case IDENT_DRAINING:
1881
			ident_drain(ident, &output, osize_p);
1882
		}
1883
		return 0;
1884
	}
1885

1886
	while (*isize_p || (ident->state == IDENT_DRAINING)) {
1887
		int ch;
1888

1889
		if (ident->state == IDENT_DRAINING) {
1890
			ident_drain(ident, &output, osize_p);
1891
			if (!*osize_p)
1892
				break;
1893
			continue;
1894
		}
1895

1896
		ch = *(input++);
1897
		(*isize_p)--;
1898

1899
		if (ident->state == IDENT_SKIPPING) {
1900
			/*
1901
			 * Skipping until '$' or LF, but keeping them
1902
			 * in case it is a foreign ident.
1903
			 */
1904
			strbuf_addch(&ident->left, ch);
1905
			if (ch != '\n' && ch != '$')
1906
				continue;
1907
			if (ch == '$' && !is_foreign_ident(ident->left.buf)) {
1908
				strbuf_setlen(&ident->left, sizeof(head) - 1);
1909
				strbuf_addstr(&ident->left, ident->ident);
1910
			}
1911
			ident->state = IDENT_DRAINING;
1912
			continue;
1913
		}
1914

1915
		if (ident->state < sizeof(head) &&
1916
		    head[ident->state] == ch) {
1917
			ident->state++;
1918
			continue;
1919
		}
1920

1921
		if (ident->state)
1922
			strbuf_add(&ident->left, head, ident->state);
1923
		if (ident->state == sizeof(head) - 1) {
1924
			if (ch != ':' && ch != '$') {
1925
				strbuf_addch(&ident->left, ch);
1926
				ident->state = 0;
1927
				continue;
1928
			}
1929

1930
			if (ch == ':') {
1931
				strbuf_addch(&ident->left, ch);
1932
				ident->state = IDENT_SKIPPING;
1933
			} else {
1934
				strbuf_addstr(&ident->left, ident->ident);
1935
				ident->state = IDENT_DRAINING;
1936
			}
1937
			continue;
1938
		}
1939

1940
		strbuf_addch(&ident->left, ch);
1941
		ident->state = IDENT_DRAINING;
1942
	}
1943
	return 0;
1944
}
1945

1946
static void ident_free_fn(struct stream_filter *filter)
1947
{
1948
	struct ident_filter *ident = (struct ident_filter *)filter;
1949
	strbuf_release(&ident->left);
1950
	free(filter);
1951
}
1952

1953
static struct stream_filter_vtbl ident_vtbl = {
1954
	.filter = ident_filter_fn,
1955
	.free = ident_free_fn,
1956
};
1957

1958
static struct stream_filter *ident_filter(const struct object_id *oid)
1959
{
1960
	struct ident_filter *ident = xmalloc(sizeof(*ident));
1961

1962
	xsnprintf(ident->ident, sizeof(ident->ident),
1963
		  ": %s $", oid_to_hex(oid));
1964
	strbuf_init(&ident->left, 0);
1965
	ident->filter.vtbl = &ident_vtbl;
1966
	ident->state = 0;
1967
	return (struct stream_filter *)ident;
1968
}
1969

1970
/*
1971
 * Return an appropriately constructed filter for the given ca, or NULL if
1972
 * the contents cannot be filtered without reading the whole thing
1973
 * in-core.
1974
 *
1975
 * Note that you would be crazy to set CRLF, smudge/clean or ident to a
1976
 * large binary blob you would want us not to slurp into the memory!
1977
 */
1978
struct stream_filter *get_stream_filter_ca(const struct conv_attrs *ca,
1979
					   const struct object_id *oid)
1980
{
1981
	struct stream_filter *filter = NULL;
1982

1983
	if (classify_conv_attrs(ca) != CA_CLASS_STREAMABLE)
1984
		return NULL;
1985

1986
	if (ca->ident)
1987
		filter = ident_filter(oid);
1988

1989
	if (output_eol(ca->crlf_action) == EOL_CRLF)
1990
		filter = cascade_filter(filter, lf_to_crlf_filter());
1991
	else
1992
		filter = cascade_filter(filter, &null_filter_singleton);
1993

1994
	return filter;
1995
}
1996

1997
struct stream_filter *get_stream_filter(struct index_state *istate,
1998
					const char *path,
1999
					const struct object_id *oid)
2000
{
2001
	struct conv_attrs ca;
2002
	convert_attrs(istate, &ca, path);
2003
	return get_stream_filter_ca(&ca, oid);
2004
}
2005

2006
void free_stream_filter(struct stream_filter *filter)
2007
{
2008
	filter->vtbl->free(filter);
2009
}
2010

2011
int stream_filter(struct stream_filter *filter,
2012
		  const char *input, size_t *isize_p,
2013
		  char *output, size_t *osize_p)
2014
{
2015
	return filter->vtbl->filter(filter, input, isize_p, output, osize_p);
2016
}
2017

2018
void init_checkout_metadata(struct checkout_metadata *meta, const char *refname,
2019
			    const struct object_id *treeish,
2020
			    const struct object_id *blob)
2021
{
2022
	memset(meta, 0, sizeof(*meta));
2023
	if (refname)
2024
		meta->refname = refname;
2025
	if (treeish)
2026
		oidcpy(&meta->treeish, treeish);
2027
	if (blob)
2028
		oidcpy(&meta->blob, blob);
2029
}
2030

2031
void clone_checkout_metadata(struct checkout_metadata *dst,
2032
			     const struct checkout_metadata *src,
2033
			     const struct object_id *blob)
2034
{
2035
	memcpy(dst, src, sizeof(*dst));
2036
	if (blob)
2037
		oidcpy(&dst->blob, blob);
2038
}
2039

2040
enum conv_attrs_classification classify_conv_attrs(const struct conv_attrs *ca)
2041
{
2042
	if (ca->drv) {
2043
		if (ca->drv->process)
2044
			return CA_CLASS_INCORE_PROCESS;
2045
		if (ca->drv->smudge || ca->drv->clean)
2046
			return CA_CLASS_INCORE_FILTER;
2047
	}
2048

2049
	if (ca->working_tree_encoding)
2050
		return CA_CLASS_INCORE;
2051

2052
	if (ca->crlf_action == CRLF_AUTO || ca->crlf_action == CRLF_AUTO_CRLF)
2053
		return CA_CLASS_INCORE;
2054

2055
	return CA_CLASS_STREAMABLE;
2056
}
2057