internal/db/sqlite/db_bench_test.go - main - githubmirror/syncthing

internal/db/sqlite/db_bench_test.go
328 строк8 KB
Jakob Borg
chore(sqlite): use normalised tables for file names and versions (#10383)
12 сен 2025, 12:27
12 сен 2025, 12:279ee208b
// Copyright (C) 2025 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package sqlite

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/syncthing/syncthing/internal/timeutil"
	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/osutil"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/rand"
)

var globalFi protocol.FileInfo

func BenchmarkUpdate(b *testing.B) {
	db, err := Open(b.TempDir())
	if err != nil {
		b.Fatal(err)
	}
	b.Cleanup(func() {
		if err := db.Close(); err != nil {
			b.Fatal(err)
		}
	})

	fs := make([]protocol.FileInfo, 100)
	t0 := time.Now()

	seed := 0
	size := 1000
	const numBlocks = 500

	fdb, err := db.getFolderDB(folderID, true)
	if err != nil {
		b.Fatal(err)
	}

	for size < 200_000 {
		for {
			local, err := db.CountLocal(folderID, protocol.LocalDeviceID)
			if err != nil {
				b.Fatal(err)
			}
			if local.Files >= size {
				break
			}
			fs := make([]protocol.FileInfo, 1000)
			for i := range fs {
				fs[i] = genFile(rand.String(24), numBlocks, 0)
			}
			if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
				b.Fatal(err)
			}
		}

		var files, blocks int
		if err := fdb.sql.QueryRowx(`SELECT count(*) FROM files`).Scan(&files); err != nil {
			b.Fatal(err)
		}
		if err := fdb.sql.QueryRowx(`SELECT count(*) FROM blocks`).Scan(&blocks); err != nil {
			b.Fatal(err)
		}

		d := time.Since(t0)
		b.Logf("t=%s, files=%d, blocks=%d, files/s=%.01f, blocks/s=%.01f", d, files, blocks, float64(files)/d.Seconds(), float64(blocks)/d.Seconds())

		b.Run(fmt.Sprintf("n=Insert100Loc/size=%d", size), func(b *testing.B) {
			for range b.N {
				for i := range fs {
					fs[i] = genFile(rand.String(24), numBlocks, 0)
				}
				if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=RepBlocks100/size=%d", size), func(b *testing.B) {
			for range b.N {
				for i := range fs {
					fs[i].Blocks = genBlocks(fs[i].Name, seed, 64)
					fs[i].Version = fs[i].Version.Update(42)
				}
				seed++
				if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=RepSame100/size=%d", size), func(b *testing.B) {
			for range b.N {
				for i := range fs {
					fs[i].Version = fs[i].Version.Update(42)
				}
				if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=Insert100Rem/size=%d", size), func(b *testing.B) {
			for range b.N {
				for i := range fs {
					fs[i].Blocks = genBlocks(fs[i].Name, seed, 64)
					fs[i].Version = fs[i].Version.Update(42)
					fs[i].Sequence = timeutil.StrictlyMonotonicNanos()
				}
				if err := db.Update(folderID, protocol.DeviceID{42}, fs); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=GetGlobal100/size=%d", size), func(b *testing.B) {
			for range b.N {
				for i := range fs {
					_, ok, err := db.GetGlobalFile(folderID, fs[i].Name)
					if err != nil {
						b.Fatal(err)
					}
					if !ok {
						b.Fatal("should exist")
					}
				}
			}
			b.ReportMetric(float64(b.N)*100.0/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=LocalSequenced/size=%d", size), func(b *testing.B) {
			count := 0
			for range b.N {
				cur, err := db.GetDeviceSequence(folderID, protocol.LocalDeviceID)
				if err != nil {
					b.Fatal(err)
				}
				it, errFn := db.AllLocalFilesBySequence(folderID, protocol.LocalDeviceID, cur-100, 0)
				for f := range it {
					count++
					globalFi = f
				}
				if err := errFn(); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=AllLocalBlocksWithHash/size=%d", size), func(b *testing.B) {
			count := 0
			for range b.N {
				it, errFn := db.AllLocalBlocksWithHash(folderID, globalFi.Blocks[0].Hash)
				for range it {
					count++
				}
				if err := errFn(); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "blocks/s")
		})

		b.Run(fmt.Sprintf("n=GetDeviceSequenceLoc/size=%d", size), func(b *testing.B) {
			for range b.N {
				_, err := db.GetDeviceSequence(folderID, protocol.LocalDeviceID)
				if err != nil {
					b.Fatal(err)
				}
			}
		})
		b.Run(fmt.Sprintf("n=GetDeviceSequenceRem/size=%d", size), func(b *testing.B) {
			for range b.N {
				_, err := db.GetDeviceSequence(folderID, protocol.DeviceID{42})
				if err != nil {
					b.Fatal(err)
				}
			}
		})

		b.Run(fmt.Sprintf("n=RemoteNeed/size=%d", size), func(b *testing.B) {
			count := 0
			for range b.N {
				it, errFn := db.AllNeededGlobalFiles(folderID, protocol.DeviceID{42}, config.PullOrderAlphabetic, 0, 0)
				for f := range it {
					count++
					globalFi = f
				}
				if err := errFn(); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "files/s")
		})

		b.Run(fmt.Sprintf("n=LocalNeed100Largest/size=%d", size), func(b *testing.B) {
			count := 0
			for range b.N {
				it, errFn := db.AllNeededGlobalFiles(folderID, protocol.LocalDeviceID, config.PullOrderLargestFirst, 100, 0)
				for f := range it {
					globalFi = f
					count++
				}
				if err := errFn(); err != nil {
					b.Fatal(err)
				}
			}
			b.ReportMetric(float64(count)/b.Elapsed().Seconds(), "files/s")
		})

		size += 1000
	}
}

func TestBenchmarkDropAllRemote(t *testing.T) {
	if testing.Short() || os.Getenv("LONG_TEST") == "" {
		t.Skip("slow test")
	}

	db, err := Open(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		if err := db.Close(); err != nil {
			t.Fatal(err)
		}
	})

	fs := make([]protocol.FileInfo, 1000)
	seq := 0
	for {
		local, err := db.CountLocal(folderID, protocol.LocalDeviceID)
		if err != nil {
			t.Fatal(err)
		}
		if local.Files >= 15_000 {
			break
		}
		for i := range fs {
			seq++
			fs[i] = genFile(rand.String(24), 64, seq)
		}
		if err := db.Update(folderID, protocol.DeviceID{42}, fs); err != nil {
			t.Fatal(err)
		}
		if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
			t.Fatal(err)
		}
	}

	t0 := time.Now()
	if err := db.DropAllFiles(folderID, protocol.DeviceID{42}); err != nil {
		t.Fatal(err)
	}
	d := time.Since(t0)
	t.Log("drop all took", d)
}

func TestBenchmarkSizeManyFilesRemotes(t *testing.T) {
	// Reports the database size for a setup with many files and many remote
	// devices each announcing every files, with fairly long file names and
	// "worst case" version vectors.

	if testing.Short() || os.Getenv("LONG_TEST") == "" {
		t.Skip("slow test")
	}

	dir := t.TempDir()
	db, err := Open(dir)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		if err := db.Close(); err != nil {
			t.Fatal(err)
		}
	})

	// This is equivalent to about 800 GiB in 100k files (i.e., 8 MiB per
	// file), shared between 31 devices where each have touched every file.
	const numFiles = 1e5
	const numRemotes = 30
	const numBlocks = 64
	const filenameLen = 64

	fs := make([]protocol.FileInfo, 1000)
	n := 0
	seq := 0
	for n < numFiles {
		for i := range fs {
			seq++
			fs[i] = genFile(rand.String(filenameLen), numBlocks, seq)
			for r := range numRemotes {
				fs[i].Version = fs[i].Version.Update(42 + protocol.ShortID(r))
			}
		}
		if err := db.Update(folderID, protocol.LocalDeviceID, fs); err != nil {
			t.Fatal(err)
		}
		for r := range numRemotes {
			if err := db.Update(folderID, protocol.DeviceID{byte(42 + r)}, fs); err != nil {
				t.Fatal(err)
			}
		}
		n += len(fs)
		t.Log(n, (numRemotes+1)*n)
	}

	if err := db.Close(); err != nil {
		t.Fatal(err)
	}

	size := osutil.DirSize(dir)
	t.Logf("Total size: %.02f MiB", float64(size)/1024/1024)
}