lib/fs/casefs.go - main - githubmirror/syncthing

lib/fs/casefs.go
579 строк14 KB
Jakob Borg
chore(fs): optimise casefs caching performance (#10830)
27 июл 2026, 13:05
27 июл 2026, 13:05b2092b1
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package fs

import (
	"context"
	"errors"
	"fmt"
	"path/filepath"
	"sync"
	"time"

	lru "github.com/hashicorp/golang-lru/v2"
	"golang.org/x/text/unicode/norm"
)

const (
	// How long to consider cached dirnames valid
	caseCacheTimeout   = time.Second
	caseCacheItemLimit = 4 << 10
)

type CaseConflictError struct {
	Given, Real string
}

func (e *CaseConflictError) Error() string {
	return fmt.Sprintf(`remote "%v" uses different upper or lowercase characters than local "%v"; change the casing on either side to match the other`, e.Given, e.Real)
}

func IsErrCaseConflict(err error) bool {
	e := &CaseConflictError{}
	return errors.As(err, &e)
}

type realCaser interface {
	realCase(name string) (string, error)
	dropCache()
	noteAdded(name string)
	noteRemoved(name string)
}

type fskey struct {
	fstype    FilesystemType
	uri, opts string
}

// caseFilesystemRegistry caches caseFilesystems and runs a routine to drop
// their cache every now and then.
type caseFilesystemRegistry struct {
	caseCaches   map[fskey]*caseCache
	mut          sync.RWMutex
	startCleaner sync.Once
}

func newFSKey(fs Filesystem) fskey {
	k := fskey{
		fstype: fs.Type(),
		uri:    fs.URI(),
	}
	if opts := fs.Options(); len(opts) > 0 {
		k.opts = opts[0].String()
		for _, o := range opts[1:] {
			k.opts += "&" + o.String()
		}
	}
	return k
}

func (r *caseFilesystemRegistry) get(fs Filesystem) Filesystem {
	k := newFSKey(fs)

	// Use double locking when getting a caseFs. In the common case it will
	// already exist and we take the read lock fast path. If it doesn't, we
	// take a write lock and try again.

	r.mut.RLock()
	cache, ok := r.caseCaches[k]
	r.mut.RUnlock()

	if !ok {
		r.mut.Lock()
		cache, ok = r.caseCaches[k]
		if !ok {
			cache = newCaseCache()
			r.caseCaches[k] = cache
			r.startCleaner.Do(func() {
				go r.cleaner()
			})
			l.Debugf("Created new case cache for key %v", k)
		}
		r.mut.Unlock()
	}
	if ok {
		l.Debugf("Reusing case cache for key %v", k)
	}

	return &caseFilesystem{
		Filesystem: fs,
		realCaser: &defaultRealCaser{
			fs:    fs,
			cache: cache,
		},
	}
}

func (r *caseFilesystemRegistry) cleaner() {
	for range time.NewTicker(time.Minute).C {
		// We need to not hold this lock for a long time, as it blocks
		// creating new filesystems in get(), which is needed to do things
		// like add new folders. The (*caseFs).dropCache() method can take
		// an arbitrarily long time to kick in because it in turn waits for
		// locks held by things performing I/O. So we can't call that from
		// within the loop.

		r.mut.RLock()
		toProcess := make([]*caseCache, 0, len(r.caseCaches))
		for _, cache := range r.caseCaches {
			toProcess = append(toProcess, cache)
		}
		r.mut.RUnlock()

		for _, cache := range toProcess {
			cache.Purge()
		}
	}
}

var globalCaseFilesystemRegistry = caseFilesystemRegistry{caseCaches: make(map[fskey]*caseCache)}

// OptionDetectCaseConflicts ensures that the potentially case-insensitive filesystem
// behaves like a case-sensitive filesystem. Meaning that it takes into account
// the real casing of a path and returns ErrCaseConflict if the given path differs
// from the real path. It is safe to use with any filesystem, i.e. also a
// case-sensitive one. However it will add some overhead and thus shouldn't be
// used if the filesystem is known to already behave case-sensitively.
type OptionDetectCaseConflicts struct{}

func (*OptionDetectCaseConflicts) apply(fs Filesystem) Filesystem {
	return globalCaseFilesystemRegistry.get(fs)
}

func (*OptionDetectCaseConflicts) String() string {
	return "detectCaseConflicts"
}

// caseFilesystem is a BasicFilesystem with additional checks to make a
// potentially case insensitive underlying FS behave like it's case-sensitive.
type caseFilesystem struct {
	Filesystem
	realCaser
}

func (f *caseFilesystem) Chmod(name string, mode FileMode) error {
	if _, err := f.checkCase(name); err != nil {
		return err
	}
	return f.Filesystem.Chmod(name, mode)
}

func (f *caseFilesystem) Lchown(name, uid, gid string) error {
	if _, err := f.checkCase(name); err != nil {
		return err
	}
	return f.Filesystem.Lchown(name, uid, gid)
}

func (f *caseFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
	if _, err := f.checkCase(name); err != nil {
		return err
	}
	return f.Filesystem.Chtimes(name, atime, mtime)
}

func (f *caseFilesystem) Mkdir(name string, perm FileMode) error {
	can, err := f.checkCase(name)
	if err != nil {
		return err
	}
	if err := f.Filesystem.Mkdir(name, perm); err != nil {
		return err
	}
	f.noteAdded(can)
	return nil
}

func (f *caseFilesystem) MkdirAll(path string, perm FileMode) error {
	if _, err := f.checkCase(path); err != nil {
		return err
	}
	if err := f.Filesystem.MkdirAll(path, perm); err != nil {
		return err
	}
	f.dropCache()
	return nil
}

func (f *caseFilesystem) Lstat(name string) (FileInfo, error) {
	var err error
	if name, err = Canonicalize(name); err != nil {
		return nil, err
	}
	stat, err := f.Filesystem.Lstat(name)
	if err != nil {
		return nil, err
	}
	if _, err = f.checkCaseExisting(name); err != nil {
		return nil, err
	}
	return stat, nil
}

func (f *caseFilesystem) Remove(name string) error {
	can, err := f.checkCase(name)
	if err != nil {
		return err
	}
	if err := f.Filesystem.Remove(name); err != nil {
		return err
	}
	f.noteRemoved(can)
	return nil
}

func (f *caseFilesystem) RemoveAll(name string) error {
	if _, err := f.checkCase(name); err != nil {
		return err
	}
	if err := f.Filesystem.RemoveAll(name); err != nil {
		return err
	}
	f.dropCache()
	return nil
}

func (f *caseFilesystem) Rename(oldpath, newpath string) error {
	oldcan, err := f.checkCase(oldpath)
	if err != nil {
		return err
	}
	newcan, err := f.checkCase(newpath)
	if err != nil {
		// Case-only rename is ok
		e := &CaseConflictError{}
		if !errors.As(err, &e) || e.Real != oldpath {
			return err
		}
	}
	if err := f.Filesystem.Rename(oldpath, newpath); err != nil {
		return err
	}
	f.noteRemoved(oldcan)
	f.noteAdded(newcan)
	return nil
}

func (f *caseFilesystem) Stat(name string) (FileInfo, error) {
	var err error
	if name, err = Canonicalize(name); err != nil {
		return nil, err
	}
	stat, err := f.Filesystem.Stat(name)
	if err != nil {
		return nil, err
	}
	if _, err = f.checkCaseExisting(name); err != nil {
		return nil, err
	}
	return stat, nil
}

func (f *caseFilesystem) DirNames(name string) ([]string, error) {
	if _, err := f.checkCase(name); err != nil {
		return nil, err
	}
	return f.Filesystem.DirNames(name)
}

func (f *caseFilesystem) Open(name string) (File, error) {
	if _, err := f.checkCase(name); err != nil {
		return nil, err
	}
	return f.Filesystem.Open(name)
}

func (f *caseFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
	can, err := f.checkCase(name)
	if err != nil {
		return nil, err
	}
	file, err := f.Filesystem.OpenFile(name, flags, mode)
	if err != nil {
		return nil, err
	}
	f.noteAdded(can)
	return file, nil
}

func (f *caseFilesystem) ReadSymlink(name string) (string, error) {
	if _, err := f.checkCase(name); err != nil {
		return "", err
	}
	return f.Filesystem.ReadSymlink(name)
}

func (f *caseFilesystem) Create(name string) (File, error) {
	can, err := f.checkCase(name)
	if err != nil {
		return nil, err
	}
	file, err := f.Filesystem.Create(name)
	if err != nil {
		return nil, err
	}
	f.noteAdded(can)
	return file, nil
}

func (f *caseFilesystem) CreateSymlink(target, name string) error {
	can, err := f.checkCase(name)
	if err != nil {
		return err
	}
	if err := f.Filesystem.CreateSymlink(target, name); err != nil {
		return err
	}
	f.noteAdded(can)
	return nil
}

func (f *caseFilesystem) Walk(root string, walkFn WalkFunc) error {
	// Walking the filesystem is likely (in Syncthing's case certainly) done
	// to pick up external changes, for which caching is undesirable.
	f.dropCache()
	if _, err := f.checkCase(root); err != nil {
		return err
	}
	return f.Filesystem.Walk(root, walkFn)
}

func (f *caseFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
	if _, err := f.checkCase(path); err != nil {
		return nil, nil, err
	}
	return f.Filesystem.Watch(path, ignore, ctx, ignorePerms)
}

func (f *caseFilesystem) Hide(name string) error {
	if _, err := f.checkCase(name); err != nil {
		return err
	}
	return f.Filesystem.Hide(name)
}

func (f *caseFilesystem) Unhide(name string) error {
	if _, err := f.checkCase(name); err != nil {
		return err
	}
	return f.Filesystem.Unhide(name)
}

func (f *caseFilesystem) underlying() (Filesystem, bool) {
	return f.Filesystem, true
}

func (f *caseFilesystem) checkCase(name string) (string, error) {
	var err error
	if name, err = Canonicalize(name); err != nil {
		return "", err
	}
	// Stat is necessary for case sensitive FS, as it's then not a conflict
	// if name is e.g. "foo" and on dir there is "Foo".
	if _, err := f.Filesystem.Lstat(name); err != nil {
		if IsNotExist(err) {
			return name, nil
		}
		return "", err
	}
	return f.checkCaseExisting(name)
}

// checkCaseExisting must only be called after successfully canonicalizing and
// stating the file.
func (f *caseFilesystem) checkCaseExisting(name string) (string, error) {
	realName, err := f.realCase(name)
	if IsNotExist(err) {
		// It did exist just before -> cache is outdated, try again
		f.dropCache()
		realName, err = f.realCase(name)
	}
	if err != nil {
		return "", err
	}
	// We normalize the normalization (hah!) of the strings before
	// comparing, as we don't want to treat a normalization difference as a
	// case conflict.
	if norm.NFC.String(realName) != norm.NFC.String(name) {
		return "", &CaseConflictError{name, realName}
	}
	return realName, nil
}

type defaultRealCaser struct {
	cache *caseCache
	fs    Filesystem
}

type caseCache struct {
	*lru.TwoQueueCache[string, *caseNode]

	mut sync.Mutex
}

func newCaseCache() *caseCache {
	cache, err := lru.New2Q[string, *caseNode](caseCacheItemLimit)
	// New2Q only errors if given invalid parameters, which we don't.
	if err != nil {
		panic(err)
	}
	return &caseCache{
		TwoQueueCache: cache,
	}
}

func (r *defaultRealCaser) realCase(name string) (string, error) {
	realName := "."
	if name == realName {
		return realName, nil
	}

	r.cache.mut.Lock()
	defer r.cache.mut.Unlock()

	for _, comp := range PathComponents(name) {
		node := r.cache.getExpireAdd(realName, r.fs)

		if node.err != nil {
			return "", node.err
		}

		// Try to find a direct or case match
		if _, ok := node.children[comp]; !ok {
			comp, ok = node.lowerToReal[UnicodeLowercaseNormalized(comp)]
			if !ok {
				return "", ErrNotExist
			}
		}

		realName = filepath.Join(realName, comp)
	}

	return realName, nil
}

func (r *defaultRealCaser) dropCache() {
	r.cache.Purge()
}

func (r *defaultRealCaser) noteAdded(canonicalName string) {
	r.cache.added(canonicalName)
}

func (r *defaultRealCaser) noteRemoved(canonicalName string) {
	r.cache.removed(canonicalName)
}

// added updates the cached listing of name's parent directory to include
// name, dropping any cached listing for name itself.
func (c *caseCache) added(name string) {
	c.mut.Lock()
	defer c.mut.Unlock()

	// Remove any stale entry for the name itself, though it should already
	// have been removed by a paired noteRemoved if it existed before.
	c.Remove(name)

	// Get and update parent

	node, ok := c.Get(filepath.Dir(name))
	if !ok || node.err != nil {
		return
	}

	base := filepath.Base(name)
	node.children[base] = struct{}{}
	node.lowerToReal[UnicodeLowercaseNormalized(base)] = base
}

// removed updates the cached listing of name's parent directory to no longer
// include name, dropping any cached listing for name itself.
func (c *caseCache) removed(name string) {
	c.mut.Lock()
	defer c.mut.Unlock()

	// Remove the cache entry for the name itself, if it exists
	c.Remove(name)

	// Get the parent and remove it from its set of children

	node, ok := c.Get(filepath.Dir(name))
	if !ok || node.err != nil {
		return
	}

	if len(node.children) != len(node.lowerToReal) {
		// Some entries share a lowercase form, only possible on a
		// case-sensitive filesystem. That makes precise incremental removal
		// awkward (we'd need to rebuild lowerToReal in sorted order etc),
		// so drop the listing and let it be re-read.
		c.Remove(filepath.Dir(name))
		return
	}

	base := filepath.Base(name)
	delete(node.children, base)
	delete(node.lowerToReal, UnicodeLowercaseNormalized(base))
}

// getExpireAdd gets an entry for the given key. If no entry exists, or it is
// expired a new one is created and added to the cache. The caller must hold
// c.mut.
func (c *caseCache) getExpireAdd(key string, fs Filesystem) *caseNode {
	node, ok := c.Get(key)
	if !ok {
		node := newCaseNode(key, fs)
		c.Add(key, node)
		return node
	}
	if node.expires.Before(time.Now()) {
		node = newCaseNode(key, fs)
		c.Add(key, node)
	}
	return node
}

// The keys to children are "real", case resolved names of the path
// component this node represents (i.e. containing no path separator).
// lowerToReal is a map of lowercase path components (as in UnicodeLowercase)
// to their corresponding "real", case resolved names.
// A node is created empty and populated using once. If an error occurs the node
// is removed from cache and the error stored in err, such that anyone that
// already got the node doesn't try to access the nil maps.
type caseNode struct {
	expires     time.Time
	lowerToReal map[string]string
	children    map[string]struct{}
	err         error
}

func newCaseNode(name string, filesystem Filesystem) *caseNode {
	node := new(caseNode)
	dirNames, err := filesystem.DirNames(name)
	// Set expiry after calling DirNames in case this is super-slow
	// (e.g. dirs with many children on android)
	node.expires = time.Now().Add(caseCacheTimeout)
	if err != nil {
		node.err = err
		return node
	}

	num := len(dirNames)
	node.children = make(map[string]struct{}, num)
	node.lowerToReal = make(map[string]string, num)
	lastLower := ""
	for _, n := range dirNames {
		node.children[n] = struct{}{}
		lower := UnicodeLowercaseNormalized(n)
		if lower != lastLower {
			node.lowerToReal[lower] = n
			lastLower = lower
		}
	}

	return node
}