git
/
sparse-index.c
742 строки · 20.0 Кб
1#include "git-compat-util.h"
2#include "environment.h"
3#include "gettext.h"
4#include "name-hash.h"
5#include "read-cache-ll.h"
6#include "repository.h"
7#include "sparse-index.h"
8#include "tree.h"
9#include "pathspec.h"
10#include "trace2.h"
11#include "cache-tree.h"
12#include "config.h"
13#include "dir.h"
14#include "fsmonitor-ll.h"
15#include "advice.h"
16
17/**
18* This global is used by expand_index() to determine if we should give the
19* advice for advice.sparseIndexExpanded when expanding a sparse index to a full
20* one. However, this is sometimes done on purpose, such as in the sparse-checkout
21* builtin, even when index.sparse=false. This may be disabled in
22* convert_to_sparse().
23*/
24static int give_advice_on_expansion = 1;
25#define ADVICE_MSG \
26"The sparse index is expanding to a full index, a slow operation.\n" \
27"Your working directory likely has contents that are outside of\n" \
28"your sparse-checkout patterns. Use 'git sparse-checkout list' to\n" \
29"see your sparse-checkout definition and compare it to your working\n" \
30"directory contents. Running 'git clean' may assist in this cleanup."
31
32struct modify_index_context {
33struct index_state *write;
34struct pattern_list *pl;
35};
36
37static struct cache_entry *construct_sparse_dir_entry(
38struct index_state *istate,
39const char *sparse_dir,
40struct cache_tree *tree)
41{
42struct cache_entry *de;
43
44de = make_cache_entry(istate, S_IFDIR, &tree->oid, sparse_dir, 0, 0);
45
46de->ce_flags |= CE_SKIP_WORKTREE;
47return de;
48}
49
50/*
51* Returns the number of entries "inserted" into the index.
52*/
53static int convert_to_sparse_rec(struct index_state *istate,
54int num_converted,
55int start, int end,
56const char *ct_path, size_t ct_pathlen,
57struct cache_tree *ct)
58{
59int i, can_convert = 1;
60int start_converted = num_converted;
61struct strbuf child_path = STRBUF_INIT;
62
63/*
64* Is the current path outside of the sparse cone?
65* Then check if the region can be replaced by a sparse
66* directory entry (everything is sparse and merged).
67*/
68if (path_in_sparse_checkout(ct_path, istate))
69can_convert = 0;
70
71for (i = start; can_convert && i < end; i++) {
72struct cache_entry *ce = istate->cache[i];
73
74if (ce_stage(ce) ||
75S_ISGITLINK(ce->ce_mode) ||
76!(ce->ce_flags & CE_SKIP_WORKTREE))
77can_convert = 0;
78}
79
80if (can_convert) {
81struct cache_entry *se;
82se = construct_sparse_dir_entry(istate, ct_path, ct);
83
84istate->cache[num_converted++] = se;
85return 1;
86}
87
88for (i = start; i < end; ) {
89int count, span, pos = -1;
90const char *base, *slash;
91struct cache_entry *ce = istate->cache[i];
92
93/*
94* Detect if this is a normal entry outside of any subtree
95* entry.
96*/
97base = ce->name + ct_pathlen;
98slash = strchr(base, '/');
99
100if (slash)
101pos = cache_tree_subtree_pos(ct, base, slash - base);
102
103if (pos < 0) {
104istate->cache[num_converted++] = ce;
105i++;
106continue;
107}
108
109strbuf_setlen(&child_path, 0);
110strbuf_add(&child_path, ce->name, slash - ce->name + 1);
111
112span = ct->down[pos]->cache_tree->entry_count;
113count = convert_to_sparse_rec(istate,
114num_converted, i, i + span,
115child_path.buf, child_path.len,
116ct->down[pos]->cache_tree);
117num_converted += count;
118i += span;
119}
120
121strbuf_release(&child_path);
122return num_converted - start_converted;
123}
124
125int set_sparse_index_config(struct repository *repo, int enable)
126{
127int res = repo_config_set_worktree_gently(repo,
128"index.sparse",
129enable ? "true" : "false");
130prepare_repo_settings(repo);
131repo->settings.sparse_index = enable;
132return res;
133}
134
135static int index_has_unmerged_entries(struct index_state *istate)
136{
137int i;
138for (i = 0; i < istate->cache_nr; i++) {
139if (ce_stage(istate->cache[i]))
140return 1;
141}
142
143return 0;
144}
145
146int is_sparse_index_allowed(struct index_state *istate, int flags)
147{
148if (!core_apply_sparse_checkout || !core_sparse_checkout_cone)
149return 0;
150
151if (!(flags & SPARSE_INDEX_MEMORY_ONLY)) {
152int test_env;
153
154/*
155* The sparse index is not (yet) integrated with a split index.
156*/
157if (istate->split_index || git_env_bool("GIT_TEST_SPLIT_INDEX", 0))
158return 0;
159/*
160* The GIT_TEST_SPARSE_INDEX environment variable triggers the
161* index.sparse config variable to be on.
162*/
163test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1);
164if (test_env >= 0)
165set_sparse_index_config(istate->repo, test_env);
166
167/*
168* Only convert to sparse if index.sparse is set.
169*/
170prepare_repo_settings(istate->repo);
171if (!istate->repo->settings.sparse_index)
172return 0;
173}
174
175if (init_sparse_checkout_patterns(istate))
176return 0;
177
178/*
179* We need cone-mode patterns to use sparse-index. If a user edits
180* their sparse-checkout file manually, then we can detect during
181* parsing that they are not actually using cone-mode patterns and
182* hence we need to abort this conversion _without error_. Warnings
183* already exist in the pattern parsing to inform the user of their
184* bad patterns.
185*/
186if (!istate->sparse_checkout_patterns->use_cone_patterns)
187return 0;
188
189return 1;
190}
191
192int convert_to_sparse(struct index_state *istate, int flags)
193{
194/*
195* If the index is already sparse, empty, or otherwise
196* cannot be converted to sparse, do not convert.
197*/
198if (istate->sparse_index == INDEX_COLLAPSED || !istate->cache_nr ||
199!is_sparse_index_allowed(istate, flags))
200return 0;
201
202/*
203* If we are purposefully collapsing a full index, then don't give
204* advice when it is expanded later.
205*/
206give_advice_on_expansion = 0;
207
208/*
209* NEEDSWORK: If we have unmerged entries, then stay full.
210* Unmerged entries prevent the cache-tree extension from working.
211*/
212if (index_has_unmerged_entries(istate))
213return 0;
214
215if (!cache_tree_fully_valid(istate->cache_tree)) {
216/* Clear and recompute the cache-tree */
217cache_tree_free(&istate->cache_tree);
218
219/*
220* Silently return if there is a problem with the cache tree update,
221* which might just be due to a conflict state in some entry.
222*
223* This might create new tree objects, so be sure to use
224* WRITE_TREE_MISSING_OK.
225*/
226if (cache_tree_update(istate, WRITE_TREE_MISSING_OK))
227return 0;
228}
229
230remove_fsmonitor(istate);
231
232trace2_region_enter("index", "convert_to_sparse", istate->repo);
233istate->cache_nr = convert_to_sparse_rec(istate,
2340, 0, istate->cache_nr,
235"", 0, istate->cache_tree);
236
237/* Clear and recompute the cache-tree */
238cache_tree_free(&istate->cache_tree);
239cache_tree_update(istate, 0);
240
241istate->fsmonitor_has_run_once = 0;
242FREE_AND_NULL(istate->fsmonitor_dirty);
243FREE_AND_NULL(istate->fsmonitor_last_update);
244
245istate->sparse_index = INDEX_COLLAPSED;
246trace2_region_leave("index", "convert_to_sparse", istate->repo);
247return 0;
248}
249
250static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
251{
252ALLOC_GROW(istate->cache, nr + 1, istate->cache_alloc);
253
254istate->cache[nr] = ce;
255add_name_hash(istate, ce);
256}
257
258static int add_path_to_index(const struct object_id *oid,
259struct strbuf *base, const char *path,
260unsigned int mode, void *context)
261{
262struct modify_index_context *ctx = (struct modify_index_context *)context;
263struct cache_entry *ce;
264size_t len = base->len;
265
266if (S_ISDIR(mode)) {
267int dtype;
268size_t baselen = base->len;
269if (!ctx->pl)
270return READ_TREE_RECURSIVE;
271
272/*
273* Have we expanded to a point outside of the sparse-checkout?
274*
275* Artificially pad the path name with a slash "/" to
276* indicate it as a directory, and add an arbitrary file
277* name ("-") so we can consider base->buf as a file name
278* to match against the cone-mode patterns.
279*
280* If we compared just "path", then we would expand more
281* than we should. Since every file at root is always
282* included, we would expand every directory at root at
283* least one level deep instead of using sparse directory
284* entries.
285*/
286strbuf_addstr(base, path);
287strbuf_add(base, "/-", 2);
288
289if (path_matches_pattern_list(base->buf, base->len,
290NULL, &dtype,
291ctx->pl, ctx->write)) {
292strbuf_setlen(base, baselen);
293return READ_TREE_RECURSIVE;
294}
295
296/*
297* The path "{base}{path}/" is a sparse directory. Create the correct
298* name for inserting the entry into the index.
299*/
300strbuf_setlen(base, base->len - 1);
301} else {
302strbuf_addstr(base, path);
303}
304
305ce = make_cache_entry(ctx->write, mode, oid, base->buf, 0, 0);
306ce->ce_flags |= CE_SKIP_WORKTREE | CE_EXTENDED;
307set_index_entry(ctx->write, ctx->write->cache_nr++, ce);
308
309strbuf_setlen(base, len);
310return 0;
311}
312
313void expand_index(struct index_state *istate, struct pattern_list *pl)
314{
315int i;
316struct index_state *full;
317struct strbuf base = STRBUF_INIT;
318const char *tr_region;
319struct modify_index_context ctx;
320
321/*
322* If the index is already full, then keep it full. We will convert
323* it to a sparse index on write, if possible.
324*/
325if (istate->sparse_index == INDEX_EXPANDED)
326return;
327
328/*
329* If our index is sparse, but our new pattern set does not use
330* cone mode patterns, then we need to expand the index before we
331* continue. A NULL pattern set indicates a full expansion to a
332* full index.
333*/
334if (pl && !pl->use_cone_patterns) {
335pl = NULL;
336} else {
337/*
338* We might contract file entries into sparse-directory
339* entries, and for that we will need the cache tree to
340* be recomputed.
341*/
342cache_tree_free(&istate->cache_tree);
343
344/*
345* If there is a problem creating the cache tree, then we
346* need to expand to a full index since we cannot satisfy
347* the current request as a sparse index.
348*/
349if (cache_tree_update(istate, 0))
350pl = NULL;
351}
352
353if (!pl && give_advice_on_expansion) {
354give_advice_on_expansion = 0;
355advise_if_enabled(ADVICE_SPARSE_INDEX_EXPANDED,
356_(ADVICE_MSG));
357}
358
359/*
360* A NULL pattern set indicates we are expanding a full index, so
361* we use a special region name that indicates the full expansion.
362* This is used by test cases, but also helps to differentiate the
363* two cases.
364*/
365tr_region = pl ? "expand_index" : "ensure_full_index";
366trace2_region_enter("index", tr_region, istate->repo);
367
368/* initialize basics of new index */
369full = xcalloc(1, sizeof(struct index_state));
370memcpy(full, istate, sizeof(struct index_state));
371
372/*
373* This slightly-misnamed 'full' index might still be sparse if we
374* are only modifying the list of sparse directories. This hinges
375* on whether we have a non-NULL pattern list.
376*/
377full->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED;
378
379/* then change the necessary things */
380full->cache_alloc = (3 * istate->cache_alloc) / 2;
381full->cache_nr = 0;
382ALLOC_ARRAY(full->cache, full->cache_alloc);
383
384ctx.write = full;
385ctx.pl = pl;
386
387for (i = 0; i < istate->cache_nr; i++) {
388struct cache_entry *ce = istate->cache[i];
389struct tree *tree;
390struct pathspec ps;
391int dtype;
392
393if (!S_ISSPARSEDIR(ce->ce_mode)) {
394set_index_entry(full, full->cache_nr++, ce);
395continue;
396}
397
398/* We now have a sparse directory entry. Should we expand? */
399if (pl &&
400path_matches_pattern_list(ce->name, ce->ce_namelen,
401NULL, &dtype,
402pl, istate) == NOT_MATCHED) {
403set_index_entry(full, full->cache_nr++, ce);
404continue;
405}
406
407if (!(ce->ce_flags & CE_SKIP_WORKTREE))
408warning(_("index entry is a directory, but not sparse (%08x)"),
409ce->ce_flags);
410
411/* recursively walk into cd->name */
412tree = lookup_tree(istate->repo, &ce->oid);
413
414memset(&ps, 0, sizeof(ps));
415ps.recursive = 1;
416ps.has_wildcard = 1;
417ps.max_depth = -1;
418
419strbuf_setlen(&base, 0);
420strbuf_add(&base, ce->name, strlen(ce->name));
421
422read_tree_at(istate->repo, tree, &base, 0, &ps,
423add_path_to_index, &ctx);
424
425/* free directory entries. full entries are re-used */
426discard_cache_entry(ce);
427}
428
429/* Copy back into original index. */
430memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash));
431memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash));
432istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED;
433free(istate->cache);
434istate->cache = full->cache;
435istate->cache_nr = full->cache_nr;
436istate->cache_alloc = full->cache_alloc;
437istate->fsmonitor_has_run_once = 0;
438FREE_AND_NULL(istate->fsmonitor_dirty);
439FREE_AND_NULL(istate->fsmonitor_last_update);
440
441strbuf_release(&base);
442free(full);
443
444/* Clear and recompute the cache-tree */
445cache_tree_free(&istate->cache_tree);
446cache_tree_update(istate, 0);
447
448trace2_region_leave("index", tr_region, istate->repo);
449}
450
451void ensure_full_index(struct index_state *istate)
452{
453if (!istate)
454BUG("ensure_full_index() must get an index!");
455expand_index(istate, NULL);
456}
457
458void ensure_correct_sparsity(struct index_state *istate)
459{
460/*
461* If the index can be sparse, make it sparse. Otherwise,
462* ensure the index is full.
463*/
464if (is_sparse_index_allowed(istate, 0))
465convert_to_sparse(istate, 0);
466else
467ensure_full_index(istate);
468}
469
470struct path_found_data {
471/**
472* The path stored in 'dir', if non-empty, corresponds to the most-
473* recent path that we checked where:
474*
475* 1. The path should be a directory, according to the index.
476* 2. The path does not exist.
477* 3. The parent path _does_ exist. (This may be the root of the
478* working directory.)
479*/
480struct strbuf dir;
481size_t lstat_count;
482};
483
484#define PATH_FOUND_DATA_INIT { \
485.dir = STRBUF_INIT \
486}
487
488static void clear_path_found_data(struct path_found_data *data)
489{
490strbuf_release(&data->dir);
491}
492
493/**
494* Return the length of the longest common substring that ends in a
495* slash ('/') to indicate the longest common parent directory. Returns
496* zero if no common directory exists.
497*/
498static size_t max_common_dir_prefix(const char *path1, const char *path2)
499{
500size_t common_prefix = 0;
501for (size_t i = 0; path1[i] && path2[i]; i++) {
502if (path1[i] != path2[i])
503break;
504
505/*
506* If they agree at a directory separator, then add one
507* to make sure it is included in the common prefix string.
508*/
509if (path1[i] == '/')
510common_prefix = i + 1;
511}
512
513return common_prefix;
514}
515
516static int path_found(const char *path, struct path_found_data *data)
517{
518struct stat st;
519size_t common_prefix;
520
521/*
522* If data->dir is non-empty, then it contains a path that doesn't
523* exist, including an ending slash ('/'). If it is a prefix of 'path',
524* then we can return 0.
525*/
526if (data->dir.len && !memcmp(path, data->dir.buf, data->dir.len))
527return 0;
528
529/*
530* Otherwise, we must check if the current path exists. If it does, then
531* return 1. The cached directory will be skipped until we come across
532* a missing path again.
533*/
534data->lstat_count++;
535if (!lstat(path, &st))
536return 1;
537
538/*
539* At this point, we know that 'path' doesn't exist, and we know that
540* the parent directory of 'data->dir' does exist. Let's set 'data->dir'
541* to be the top-most non-existing directory of 'path'. If the first
542* parent of 'path' exists, then we will act as though 'path'
543* corresponds to a directory (by adding a slash).
544*/
545common_prefix = max_common_dir_prefix(path, data->dir.buf);
546
547/*
548* At this point, 'path' and 'data->dir' have a common existing parent
549* directory given by path[0..common_prefix] (which could have length 0).
550* We "grow" the data->dir buffer by checking for existing directories
551* along 'path'.
552*/
553
554strbuf_setlen(&data->dir, common_prefix);
555while (1) {
556/* Find the next directory in 'path'. */
557const char *rest = path + data->dir.len;
558const char *next_slash = strchr(rest, '/');
559
560/*
561* If there are no more slashes, then 'path' doesn't contain a
562* non-existent _parent_ directory. Set 'data->dir' to be equal
563* to 'path' plus an additional slash, so it can be used for
564* caching in the future. The filename of 'path' is considered
565* a non-existent directory.
566*
567* Note: if "{path}/" exists as a directory, then it will never
568* appear as a prefix of other callers to this method, assuming
569* the context from the clear_skip_worktree... methods. If this
570* method is reused, then this must be reconsidered.
571*/
572if (!next_slash) {
573strbuf_addstr(&data->dir, rest);
574strbuf_addch(&data->dir, '/');
575break;
576}
577
578/*
579* Now that we have a slash, let's grow 'data->dir' to include
580* this slash, then test if we should stop.
581*/
582strbuf_add(&data->dir, rest, next_slash - rest + 1);
583
584/* If the parent dir doesn't exist, then stop here. */
585data->lstat_count++;
586if (lstat(data->dir.buf, &st))
587return 0;
588}
589
590/*
591* At this point, 'data->dir' is equal to 'path' plus a slash character,
592* and the parent directory of 'path' definitely exists. Moreover, we
593* know that 'path' doesn't exist, or we would have returned 1 earlier.
594*/
595return 0;
596}
597
598static int clear_skip_worktree_from_present_files_sparse(struct index_state *istate)
599{
600struct path_found_data data = PATH_FOUND_DATA_INIT;
601
602int path_count = 0;
603int to_restart = 0;
604
605trace2_region_enter("index", "clear_skip_worktree_from_present_files_sparse",
606istate->repo);
607for (int i = 0; i < istate->cache_nr; i++) {
608struct cache_entry *ce = istate->cache[i];
609
610if (ce_skip_worktree(ce)) {
611path_count++;
612if (path_found(ce->name, &data)) {
613if (S_ISSPARSEDIR(ce->ce_mode)) {
614to_restart = 1;
615break;
616}
617ce->ce_flags &= ~CE_SKIP_WORKTREE;
618}
619}
620}
621
622trace2_data_intmax("index", istate->repo,
623"sparse_path_count", path_count);
624trace2_data_intmax("index", istate->repo,
625"sparse_lstat_count", data.lstat_count);
626trace2_region_leave("index", "clear_skip_worktree_from_present_files_sparse",
627istate->repo);
628clear_path_found_data(&data);
629return to_restart;
630}
631
632static void clear_skip_worktree_from_present_files_full(struct index_state *istate)
633{
634struct path_found_data data = PATH_FOUND_DATA_INIT;
635
636int path_count = 0;
637
638trace2_region_enter("index", "clear_skip_worktree_from_present_files_full",
639istate->repo);
640for (int i = 0; i < istate->cache_nr; i++) {
641struct cache_entry *ce = istate->cache[i];
642
643if (S_ISSPARSEDIR(ce->ce_mode))
644BUG("ensure-full-index did not fully flatten?");
645
646if (ce_skip_worktree(ce)) {
647path_count++;
648if (path_found(ce->name, &data))
649ce->ce_flags &= ~CE_SKIP_WORKTREE;
650}
651}
652
653trace2_data_intmax("index", istate->repo,
654"full_path_count", path_count);
655trace2_data_intmax("index", istate->repo,
656"full_lstat_count", data.lstat_count);
657trace2_region_leave("index", "clear_skip_worktree_from_present_files_full",
658istate->repo);
659clear_path_found_data(&data);
660}
661
662void clear_skip_worktree_from_present_files(struct index_state *istate)
663{
664if (!core_apply_sparse_checkout ||
665sparse_expect_files_outside_of_patterns)
666return;
667
668if (clear_skip_worktree_from_present_files_sparse(istate)) {
669ensure_full_index(istate);
670clear_skip_worktree_from_present_files_full(istate);
671}
672}
673
674/*
675* This static global helps avoid infinite recursion between
676* expand_to_path() and index_file_exists().
677*/
678static int in_expand_to_path = 0;
679
680void expand_to_path(struct index_state *istate,
681const char *path, size_t pathlen, int icase)
682{
683struct strbuf path_mutable = STRBUF_INIT;
684size_t substr_len;
685
686/* prevent extra recursion */
687if (in_expand_to_path)
688return;
689
690if (!istate->sparse_index)
691return;
692
693in_expand_to_path = 1;
694
695/*
696* We only need to actually expand a region if the
697* following are both true:
698*
699* 1. 'path' is not already in the index.
700* 2. Some parent directory of 'path' is a sparse directory.
701*/
702
703if (index_file_exists(istate, path, pathlen, icase))
704goto cleanup;
705
706strbuf_add(&path_mutable, path, pathlen);
707strbuf_addch(&path_mutable, '/');
708
709/* Check the name hash for all parent directories */
710substr_len = 0;
711while (substr_len < pathlen) {
712char temp;
713char *replace = strchr(path_mutable.buf + substr_len, '/');
714
715if (!replace)
716break;
717
718/* replace the character _after_ the slash */
719replace++;
720temp = *replace;
721*replace = '\0';
722substr_len = replace - path_mutable.buf;
723if (index_file_exists(istate, path_mutable.buf,
724substr_len, icase)) {
725/*
726* We found a parent directory in the name-hash
727* hashtable, because only sparse directory entries
728* have a trailing '/' character. Since "path" wasn't
729* in the index, perhaps it exists within this
730* sparse-directory. Expand accordingly.
731*/
732ensure_full_index(istate);
733break;
734}
735
736*replace = temp;
737}
738
739cleanup:
740strbuf_release(&path_mutable);
741in_expand_to_path = 0;
742}
743