gitea

1
// Copyright 2014 The Gogs Authors. All rights reserved.
2
// Copyright 2016 The Gitea Authors. All rights reserved.
3
// SPDX-License-Identifier: MIT
4

5
package cmd
6

7
import (
8
	"fmt"
9
	"os"
10
	"path"
11
	"path/filepath"
12
	"strings"
13

14
	"code.gitea.io/gitea/models/db"
15
	"code.gitea.io/gitea/modules/dump"
16
	"code.gitea.io/gitea/modules/json"
17
	"code.gitea.io/gitea/modules/log"
18
	"code.gitea.io/gitea/modules/setting"
19
	"code.gitea.io/gitea/modules/storage"
20
	"code.gitea.io/gitea/modules/util"
21

22
	"gitea.com/go-chi/session"
23
	"github.com/mholt/archiver/v3"
24
	"github.com/urfave/cli/v2"
25
)
26

27
// CmdDump represents the available dump sub-command.
28
var CmdDump = &cli.Command{
29
	Name:        "dump",
30
	Usage:       "Dump Gitea files and database",
31
	Description: `Dump compresses all related files and database into zip file. It can be used for backup and capture Gitea server image to send to maintainer`,
32
	Action:      runDump,
33
	Flags: []cli.Flag{
34
		&cli.StringFlag{
35
			Name:    "file",
36
			Aliases: []string{"f"},
37
			Usage:   `Name of the dump file which will be created, default to "gitea-dump-{time}.zip". Supply '-' for stdout. See type for available types.`,
38
		},
39
		&cli.BoolFlag{
40
			Name:    "verbose",
41
			Aliases: []string{"V"},
42
			Usage:   "Show process details",
43
		},
44
		&cli.BoolFlag{
45
			Name:    "quiet",
46
			Aliases: []string{"q"},
47
			Usage:   "Only display warnings and errors",
48
		},
49
		&cli.StringFlag{
50
			Name:    "tempdir",
51
			Aliases: []string{"t"},
52
			Value:   os.TempDir(),
53
			Usage:   "Temporary dir path",
54
		},
55
		&cli.StringFlag{
56
			Name:    "database",
57
			Aliases: []string{"d"},
58
			Usage:   "Specify the database SQL syntax: sqlite3, mysql, mssql, postgres",
59
		},
60
		&cli.BoolFlag{
61
			Name:    "skip-repository",
62
			Aliases: []string{"R"},
63
			Usage:   "Skip the repository dumping",
64
		},
65
		&cli.BoolFlag{
66
			Name:    "skip-log",
67
			Aliases: []string{"L"},
68
			Usage:   "Skip the log dumping",
69
		},
70
		&cli.BoolFlag{
71
			Name:  "skip-custom-dir",
72
			Usage: "Skip custom directory",
73
		},
74
		&cli.BoolFlag{
75
			Name:  "skip-lfs-data",
76
			Usage: "Skip LFS data",
77
		},
78
		&cli.BoolFlag{
79
			Name:  "skip-attachment-data",
80
			Usage: "Skip attachment data",
81
		},
82
		&cli.BoolFlag{
83
			Name:  "skip-package-data",
84
			Usage: "Skip package data",
85
		},
86
		&cli.BoolFlag{
87
			Name:  "skip-index",
88
			Usage: "Skip bleve index data",
89
		},
90
		&cli.BoolFlag{
91
			Name:  "skip-db",
92
			Usage: "Skip database",
93
		},
94
		&cli.StringFlag{
95
			Name:  "type",
96
			Usage: fmt.Sprintf(`Dump output format, default to "zip", supported types: %s`, strings.Join(dump.SupportedOutputTypes, ", ")),
97
		},
98
	},
99
}
100

101
func fatal(format string, args ...any) {
102
	log.Fatal(format, args...)
103
}
104

105
func runDump(ctx *cli.Context) error {
106
	setting.MustInstalled()
107

108
	quite := ctx.Bool("quiet")
109
	verbose := ctx.Bool("verbose")
110
	if verbose && quite {
111
		fatal("Option --quiet and --verbose cannot both be set")
112
	}
113

114
	// outFileName is either "-" or a file name (will be made absolute)
115
	outFileName, outType := dump.PrepareFileNameAndType(ctx.String("file"), ctx.String("type"))
116
	if outType == "" {
117
		fatal("Invalid output type")
118
	}
119

120
	outFile := os.Stdout
121
	if outFileName != "-" {
122
		var err error
123
		if outFileName, err = filepath.Abs(outFileName); err != nil {
124
			fatal("Unable to get absolute path of dump file: %v", err)
125
		}
126
		if exist, _ := util.IsExist(outFileName); exist {
127
			fatal("Dump file %q exists", outFileName)
128
		}
129
		if outFile, err = os.Create(outFileName); err != nil {
130
			fatal("Unable to create dump file %q: %v", outFileName, err)
131
		}
132
		defer outFile.Close()
133
	}
134

135
	setupConsoleLogger(util.Iif(quite, log.WARN, log.INFO), log.CanColorStderr, os.Stderr)
136

137
	setting.DisableLoggerInit()
138
	setting.LoadSettings() // cannot access session settings otherwise
139

140
	stdCtx, cancel := installSignals()
141
	defer cancel()
142

143
	err := db.InitEngine(stdCtx)
144
	if err != nil {
145
		return err
146
	}
147

148
	if err = storage.Init(); err != nil {
149
		return err
150
	}
151

152
	archiverGeneric, err := archiver.ByExtension("." + outType)
153
	if err != nil {
154
		fatal("Unable to get archiver for extension: %v", err)
155
	}
156

157
	archiverWriter := archiverGeneric.(archiver.Writer)
158
	if err := archiverWriter.Create(outFile); err != nil {
159
		fatal("Creating archiver.Writer failed: %v", err)
160
	}
161
	defer archiverWriter.Close()
162

163
	dumper := &dump.Dumper{
164
		Writer:  archiverWriter,
165
		Verbose: verbose,
166
	}
167
	dumper.GlobalExcludeAbsPath(outFileName)
168

169
	if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
170
		log.Info("Skip dumping local repositories")
171
	} else {
172
		log.Info("Dumping local repositories... %s", setting.RepoRootPath)
173
		if err := dumper.AddRecursiveExclude("repos", setting.RepoRootPath, nil); err != nil {
174
			fatal("Failed to include repositories: %v", err)
175
		}
176

177
		if ctx.IsSet("skip-lfs-data") && ctx.Bool("skip-lfs-data") {
178
			log.Info("Skip dumping LFS data")
179
		} else if !setting.LFS.StartServer {
180
			log.Info("LFS isn't enabled. Skip dumping LFS data")
181
		} else if err := storage.LFS.IterateObjects("", func(objPath string, object storage.Object) error {
182
			info, err := object.Stat()
183
			if err != nil {
184
				return err
185
			}
186
			return dumper.AddReader(object, info, path.Join("data", "lfs", objPath))
187
		}); err != nil {
188
			fatal("Failed to dump LFS objects: %v", err)
189
		}
190
	}
191

192
	if ctx.Bool("skip-db") {
193
		// Ensure that we don't dump the database file that may reside in setting.AppDataPath or elsewhere.
194
		dumper.GlobalExcludeAbsPath(setting.Database.Path)
195
		log.Info("Skipping database")
196
	} else {
197
		tmpDir := ctx.String("tempdir")
198
		if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
199
			fatal("Path does not exist: %s", tmpDir)
200
		}
201

202
		dbDump, err := os.CreateTemp(tmpDir, "gitea-db.sql")
203
		if err != nil {
204
			fatal("Failed to create tmp file: %v", err)
205
		}
206
		defer func() {
207
			_ = dbDump.Close()
208
			if err := util.Remove(dbDump.Name()); err != nil {
209
				log.Warn("Unable to remove temporary file: %s: Error: %v", dbDump.Name(), err)
210
			}
211
		}()
212

213
		targetDBType := ctx.String("database")
214
		if len(targetDBType) > 0 && targetDBType != setting.Database.Type.String() {
215
			log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
216
		} else {
217
			log.Info("Dumping database...")
218
		}
219

220
		if err := db.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
221
			fatal("Failed to dump database: %v", err)
222
		}
223

224
		if err = dumper.AddFile("gitea-db.sql", dbDump.Name()); err != nil {
225
			fatal("Failed to include gitea-db.sql: %v", err)
226
		}
227
	}
228

229
	log.Info("Adding custom configuration file from %s", setting.CustomConf)
230
	if err = dumper.AddFile("app.ini", setting.CustomConf); err != nil {
231
		fatal("Failed to include specified app.ini: %v", err)
232
	}
233

234
	if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
235
		log.Info("Skipping custom directory")
236
	} else {
237
		customDir, err := os.Stat(setting.CustomPath)
238
		if err == nil && customDir.IsDir() {
239
			if is, _ := dump.IsSubdir(setting.AppDataPath, setting.CustomPath); !is {
240
				if err := dumper.AddRecursiveExclude("custom", setting.CustomPath, nil); err != nil {
241
					fatal("Failed to include custom: %v", err)
242
				}
243
			} else {
244
				log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
245
			}
246
		} else {
247
			log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
248
		}
249
	}
250

251
	isExist, err := util.IsExist(setting.AppDataPath)
252
	if err != nil {
253
		log.Error("Unable to check if %s exists. Error: %v", setting.AppDataPath, err)
254
	}
255
	if isExist {
256
		log.Info("Packing data directory...%s", setting.AppDataPath)
257

258
		var excludes []string
259
		if setting.SessionConfig.OriginalProvider == "file" {
260
			var opts session.Options
261
			if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
262
				return err
263
			}
264
			excludes = append(excludes, opts.ProviderConfig)
265
		}
266

267
		if ctx.IsSet("skip-index") && ctx.Bool("skip-index") {
268
			excludes = append(excludes, setting.Indexer.RepoPath)
269
			excludes = append(excludes, setting.Indexer.IssuePath)
270
		}
271

272
		excludes = append(excludes, setting.RepoRootPath)
273
		excludes = append(excludes, setting.LFS.Storage.Path)
274
		excludes = append(excludes, setting.Attachment.Storage.Path)
275
		excludes = append(excludes, setting.Packages.Storage.Path)
276
		excludes = append(excludes, setting.Log.RootPath)
277
		if err := dumper.AddRecursiveExclude("data", setting.AppDataPath, excludes); err != nil {
278
			fatal("Failed to include data directory: %v", err)
279
		}
280
	}
281

282
	if ctx.IsSet("skip-attachment-data") && ctx.Bool("skip-attachment-data") {
283
		log.Info("Skip dumping attachment data")
284
	} else if err := storage.Attachments.IterateObjects("", func(objPath string, object storage.Object) error {
285
		info, err := object.Stat()
286
		if err != nil {
287
			return err
288
		}
289
		return dumper.AddReader(object, info, path.Join("data", "attachments", objPath))
290
	}); err != nil {
291
		fatal("Failed to dump attachments: %v", err)
292
	}
293

294
	if ctx.IsSet("skip-package-data") && ctx.Bool("skip-package-data") {
295
		log.Info("Skip dumping package data")
296
	} else if !setting.Packages.Enabled {
297
		log.Info("Packages isn't enabled. Skip dumping package data")
298
	} else if err := storage.Packages.IterateObjects("", func(objPath string, object storage.Object) error {
299
		info, err := object.Stat()
300
		if err != nil {
301
			return err
302
		}
303
		return dumper.AddReader(object, info, path.Join("data", "packages", objPath))
304
	}); err != nil {
305
		fatal("Failed to dump packages: %v", err)
306
	}
307

308
	// Doesn't check if LogRootPath exists before processing --skip-log intentionally,
309
	// ensuring that it's clear the dump is skipped whether the directory's initialized
310
	// yet or not.
311
	if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
312
		log.Info("Skip dumping log files")
313
	} else {
314
		isExist, err := util.IsExist(setting.Log.RootPath)
315
		if err != nil {
316
			log.Error("Unable to check if %s exists. Error: %v", setting.Log.RootPath, err)
317
		}
318
		if isExist {
319
			if err := dumper.AddRecursiveExclude("log", setting.Log.RootPath, nil); err != nil {
320
				fatal("Failed to include log: %v", err)
321
			}
322
		}
323
	}
324

325
	if outFileName == "-" {
326
		log.Info("Finish dumping to stdout")
327
	} else {
328
		if err = archiverWriter.Close(); err != nil {
329
			_ = os.Remove(outFileName)
330
			fatal("Failed to save %q: %v", outFileName, err)
331
		}
332
		if err = os.Chmod(outFileName, 0o600); err != nil {
333
			log.Info("Can't change file access permissions mask to 0600: %v", err)
334
		}
335
		log.Info("Finish dumping in file %s", outFileName)
336
	}
337
	return nil
338
}
339