cache/filecache/filecache.go - master - githubmirror/hugo

cache/filecache/filecache.go
570 строк12 KB
Bjørn Erik Pedersen
all: Replace RWMutex struct caches with ConcurrentMap
24 май 2026, 16:29
24 май 2026, 16:29c4bbc28
// Copyright 2026 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filecache

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/gohugoio/httpcache"
	"github.com/gohugoio/hugo/common/hugio"
	"github.com/gohugoio/hugo/hugofs"

	"github.com/gohugoio/hugo/helpers"

	"github.com/BurntSushi/locker"
	"github.com/bep/helpers/maphelpers"
	"github.com/spf13/afero"
)

// ErrFatal can be used to signal an unrecoverable error.
var ErrFatal = errors.New("fatal filecache error")

const (
	FilecacheRootDirname = "filecache"
)

// Cache caches a set of files in a directory. This is usually a file on
// disk, but since this is backed by an Afero file system, it can be anything.
type Cache struct {
	Fs afero.Fs

	cfg FileCacheConfig

	entryLocker *lockTracker

	initOnce sync.Once
	isInited bool
	initErr  error
}

type lockTracker struct {
	seen *maphelpers.ConcurrentSet[string]

	*locker.Locker
}

// Lock tracks the ids in use. We use this information to do garbage collection
// after a Hugo build.
func (l *lockTracker) Lock(id string) {
	l.seen.AddIfAbsent(id)
	l.Locker.Lock(id)
}

// ItemInfo contains info about a cached file.
type ItemInfo struct {
	// This is the file's name relative to the cache's filesystem.
	Name string
}

// NewCache creates a new file cache with the given filesystem and max age.
func NewCache(fs afero.Fs, cfg FileCacheConfig) *Cache {
	if err := cfg.init(); err != nil {
		panic(fmt.Sprintf("invalid cache config: %s", err))
	}

	return &Cache{
		Fs:          fs,
		entryLocker: &lockTracker{Locker: locker.NewLocker(), seen: maphelpers.NewConcurrentSet[string]()},
		cfg:         cfg,
	}
}

// lockedFile is a file with a lock that is released on Close.
type lockedFile struct {
	afero.File
	unlock func()
}

func (l *lockedFile) Close() error {
	defer l.unlock()
	return l.File.Close()
}

func (c *Cache) init() error {
	if c == nil {
		panic("cache is nil")
	}

	c.initOnce.Do(func() {
		c.isInited = true
		// Create the base dir if it does not exist.
		if err := c.Fs.MkdirAll("", 0o777); err != nil && !os.IsExist(err) {
			err = fmt.Errorf("failled to create base cache directory: %s", err)
			c.initErr = err
		}
	})
	return c.initErr
}

// WriteCloser returns a transactional writer into the cache.
// It's important that it's closed when done.
func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, nil, err
	}

	id = cleanID(id)
	c.entryLocker.Lock(id)

	info := ItemInfo{Name: id}

	f, err := helpers.OpenFileForWriting(c.Fs, id)
	if err != nil {
		c.entryLocker.Unlock(id)
		return info, nil, err
	}

	return info, &lockedFile{
		File:   f,
		unlock: func() { c.entryLocker.Unlock(id) },
	}, nil
}

// ReadOrCreate tries to lookup the file in cache.
// If found, it is passed to read and then closed.
// If not found a new file is created and passed to create, which should close
// it when done.
func (c *Cache) ReadOrCreate(id string,
	read func(info ItemInfo, r io.ReadSeeker) error,
	create func(info ItemInfo, w io.WriteCloser) error,
) (info ItemInfo, err error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, err
	}

	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	info = ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		err = read(info, r)
		defer r.Close()
		if err == nil || err == ErrFatal {
			// See https://github.com/gohugoio/hugo/issues/6401
			// To recover from file corruption we handle read errors
			// as the cache item was not found.
			// Any file permission issue will also fail in the next step.
			return
		}
	}

	f, err := helpers.OpenFileForWriting(c.Fs, id)
	if err != nil {
		return
	}

	err = create(info, f)

	return
}

// NamedLock locks the given id. The lock is released when the returned function is called.
func (c *Cache) NamedLock(id string) func() {
	id = cleanID(id)
	c.entryLocker.Lock(id)
	return func() {
		c.entryLocker.Unlock(id)
	}
}

// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, nil, err
	}
	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	info := ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		return info, r, nil
	}

	var (
		r   io.ReadCloser
		err error
	)

	r, err = create()
	if err != nil {
		return info, nil, err
	}

	if c.cfg.MaxAge == 0 {
		// No caching.
		return info, hugio.ToReadCloser(r), nil
	}

	var buff bytes.Buffer
	return info,
		hugio.ToReadCloser(&buff),
		c.writeReader(id, io.TeeReader(r, &buff))
}

// AbsFilenameFromID returns the filename for the given id in the cache.
// This will be an absolute path.
func (c *Cache) AbsFilenameFromID(id string) string {
	return filepath.Join(c.cfg.DirCompiled, cleanID(id))
}

// GetOrCreateInfo tries to get the item info with the given id from cache. If not found or expired, create will
// be invoked with the id. The create function is expected to create the cache item with the given id. The returned ItemInfo will have the id as Name.
// This method is protected by a named lock using the given id as identifier.
func (c *Cache) GetOrCreateInfo(id string, create func(id string) error) (ItemInfo, error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, err
	}

	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	info := ItemInfo{Name: id}

	if !c.removeIfNeeded(id) {
		// The file exists and is not expired, so we consider it a cache hit.
		return info, nil
	}

	if err := create(id); err != nil {
		c.remove(id)
		return info, err
	}

	return info, nil
}

func (c *Cache) writeReader(id string, r io.Reader) error {
	dir := filepath.Dir(id)
	if dir != "" {
		_ = c.Fs.MkdirAll(dir, 0o777)
	}
	f, err := c.Fs.Create(id)
	if err != nil {
		return err
	}
	defer f.Close()

	_, _ = io.Copy(f, r)

	return nil
}

// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, nil, err
	}
	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	info := ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		defer r.Close()
		b, err := io.ReadAll(r)
		return info, b, err
	}

	var (
		b   []byte
		err error
	)

	b, err = create()
	if err != nil {
		return info, nil, err
	}

	if c.cfg.MaxAge == 0 {
		return info, b, nil
	}

	if err := c.writeReader(id, bytes.NewReader(b)); err != nil {
		return info, nil, err
	}

	return info, b, nil
}

// SetBytes sets the file content with the given id in the cache.
func (c *Cache) SetBytes(id string, data []byte) error {
	if err := c.init(); err != nil {
		return err
	}
	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	if c.cfg.MaxAge == 0 {
		// No caching.
		return nil
	}

	return c.writeReader(id, bytes.NewReader(data))
}

// GetBytes gets the file content with the given id from the cache, nil if none found.
func (c *Cache) GetBytes(id string) ([]byte, error) {
	_, b, err := c.GetItemBytes(id)
	return b, err
}

// GetItemBytes gets the ItemInfo and file content with the given id from the cache, nil if none found.
func (c *Cache) GetItemBytes(id string) (ItemInfo, []byte, error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, nil, err
	}
	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	info := ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		defer r.Close()
		b, err := io.ReadAll(r)
		return info, b, err
	}

	return info, nil, nil
}

// Get gets the file with the given id from the cache, nil if none found.
func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
	if err := c.init(); err != nil {
		return ItemInfo{}, nil, err
	}
	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	info := ItemInfo{Name: id}

	r := c.getOrRemove(id)

	return info, r, nil
}

// removeIfNeeded checks if the file with the given id should be re-created.
func (c *Cache) removeIfNeeded(id string) bool {
	if c.cfg.MaxAge == 0 {
		// No caching, remove.
		c.remove(id)
		return true
	}
	if removed, err := c.removeIfExpired(id); err != nil || removed {
		return true
	}
	return false
}

// getOrRemove gets the file with the given id. If it's expired, it will
// be removed.
func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
	if c.cfg.MaxAge == 0 {
		// No caching.
		return nil
	}

	if removed, err := c.removeIfExpired(id); err != nil || removed {
		return nil
	}

	f, err := c.Fs.Open(id)
	if err != nil {
		return nil
	}

	return f
}

func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
	if c.cfg.MaxAge == 0 {
		// No caching.
		return nil, false
	}

	f, err := c.Fs.Open(id)
	if err != nil {
		return nil, false
	}
	defer f.Close()

	b, err := io.ReadAll(f)
	if err != nil {
		return nil, false
	}

	removed, err := c.removeIfExpired(id)
	if err != nil {
		return nil, false
	}

	return b, removed
}

func (c *Cache) removeIfExpired(id string) (bool, error) {
	if c.cfg.MaxAge <= 0 {
		return false, nil
	}

	fi, err := c.Fs.Stat(id)
	if err != nil {
		return false, err
	}

	if c.isExpired(fi.ModTime()) {
		c.remove(id)
		return true, nil
	}

	return false, nil
}

func (c *Cache) remove(id string) {
	if c.cfg.entryIsDir {
		c.Fs.RemoveAll(id)
	} else {
		c.Fs.Remove(id)
	}
}

func (c *Cache) isExpired(modTime time.Time) bool {
	if c.cfg.MaxAge < 0 {
		return false
	}

	// Note the use of time.Since here.
	// We cannot use Hugo's global Clock for this.
	return c.cfg.MaxAge == 0 || time.Since(modTime) > c.cfg.MaxAge
}

// For testing
func (c *Cache) GetString(id string) string {
	id = cleanID(id)

	c.entryLocker.Lock(id)
	defer c.entryLocker.Unlock(id)

	f, err := c.Fs.Open(id)
	if err != nil {
		return ""
	}
	defer f.Close()

	b, _ := io.ReadAll(f)
	return string(b)
}

// Caches is a named set of caches.
type Caches map[string]*Cache

func (f Caches) SetResourceFs(fs afero.Fs) {
	for _, c := range f {
		if c.cfg.IsResourceDir {
			if c.isInited {
				panic("cannot set resource fs after init")
			}
			c.Fs = hugofs.NewBasePathFs(fs, c.cfg.DirCompiled)
		}
	}
}

// Get gets a named cache, nil if none found.
func (f Caches) Get(name string) *Cache {
	return f[strings.ToLower(name)]
}

// NewCaches creates a new set of file caches from the given
// configuration.
func NewCaches(dcfg Configs, sourceFs afero.Fs) (Caches, error) {
	fs := sourceFs

	m := make(Caches)
	for k, v := range dcfg {
		var cfs afero.Fs
		if v.IsResourceDir {
			cfs = nil // Set later. TODO(bep) this needs to be cleanded up.
		} else {
			cfs = hugofs.NewBasePathFs(fs, v.DirCompiled)
		}

		c := NewCache(cfs, v)

		m[k] = c
	}

	return m, nil
}

func cleanID(name string) string {
	return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
}

// AsHTTPCache returns an httpcache.Cache implementation for this file cache.
// Note that none of the methods are protected by named locks, so you need to make sure
// to do that in your own code.
func (c *Cache) AsHTTPCache() httpcache.Cache {
	return &httpCache{c: c}
}

type httpCache struct {
	c *Cache
}

func (h *httpCache) Get(id string) (resp []byte, ok bool) {
	id = cleanID(id)
	b, removed := h.c.getBytesAndRemoveIfExpired(id)

	return b, !removed
}

func (h *httpCache) Set(id string, resp []byte) {
	if h.c.cfg.MaxAge == 0 {
		return
	}

	id = cleanID(id)

	if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil {
		panic(err)
	}
}

func (h *httpCache) Delete(key string) {
	h.c.Fs.Remove(key)
}