Dragonfly2

1
/*
2
 *     Copyright 2020 The Dragonfly Authors
3
 *
4
 * Licensed under the Apache License, Version 2.0 (the "License");
5
 * you may not use this file except in compliance with the License.
6
 * You may obtain a copy of the License at
7
 *
8
 *      http://www.apache.org/licenses/LICENSE-2.0
9
 *
10
 * Unless required by applicable law or agreed to in writing, software
11
 * distributed under the License is distributed on an "AS IS" BASIS,
12
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
 * See the License for the specific language governing permissions and
14
 * limitations under the License.
15
 */
16

17
// Package config holds all DaemonConfig of dfget.
18
package config
19

20
import (
21
	"encoding/json"
22
	"fmt"
23
	"os"
24
	"path"
25
	"path/filepath"
26
	"regexp"
27
	"strings"
28
	"syscall"
29
	"time"
30

31
	"d7y.io/dragonfly/v2/client/util"
32
	"d7y.io/dragonfly/v2/cmd/dependency/base"
33
	"d7y.io/dragonfly/v2/internal/dferrors"
34
	logger "d7y.io/dragonfly/v2/internal/dflog"
35
	"d7y.io/dragonfly/v2/pkg/net/url"
36
	"d7y.io/dragonfly/v2/pkg/os/user"
37
	pkgstrings "d7y.io/dragonfly/v2/pkg/strings"
38
	"d7y.io/dragonfly/v2/pkg/unit"
39
)
40

41
type DfgetConfig = ClientOption
42

43
// ClientOption holds all the runtime config information.
44
type ClientOption struct {
45
	base.Options `yaml:",inline" mapstructure:",squash"`
46
	// URL download URL.
47
	URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"`
48

49
	// Output full output path.
50
	Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"`
51

52
	// Timeout download timeout(second).
53
	Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"`
54

55
	BenchmarkRate unit.Bytes `yaml:"benchmarkRate,omitempty" mapstructure:"benchmarkRate,omitempty"`
56

57
	// Md5 expected file md5.
58
	// Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead
59
	Md5    string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"`
60
	Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"`
61
	// DigestMethod indicates digest method, like md5, sha256
62
	DigestMethod string `yaml:"digestMethod,omitempty" mapstructure:"digestMethod,omitempty"`
63

64
	// DigestValue indicates digest value
65
	DigestValue string `yaml:"digestValue,omitempty" mapstructure:"digestValue,omitempty"`
66

67
	// Tag identify download task, it is available merely when md5 param not exist.
68
	Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"`
69

70
	// Application name that executes dfget.
71
	Application string `yaml:"application,omitempty" mapstructure:"application,omitempty"`
72

73
	// DaemonSock is daemon download socket path.
74
	DaemonSock string `yaml:"daemonSock,omitempty" mapstructure:"daemon-sock,omitempty"`
75

76
	// Priority scheduler will schedule tasks according to priority
77
	Priority int32 `yaml:"priority,omitempty" mapstructure:"priority,omitempty"`
78

79
	// CA certificate to verify when supernode interact with the source.
80
	Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"`
81

82
	// Filter filter some query params of url, use char '&' to separate different params.
83
	// eg: -f 'key&sign' will filter 'key' and 'sign' query param.
84
	// in this way, different urls correspond one same download task that can use p2p mode.
85
	Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"`
86

87
	// Header of http request.
88
	// eg: --header='Accept: *' --header='Host: abc'.
89
	Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"`
90

91
	// DisableBackSource indicates whether to not back source to download when p2p fails.
92
	DisableBackSource bool `yaml:"disableBackSource,omitempty" mapstructure:"disable-back-source,omitempty"`
93

94
	// Insecure indicates whether skip secure verify when supernode interact with the source.
95
	Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"`
96

97
	// ShowProgress shows progress bar, it's conflict with `--console`.
98
	ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"`
99

100
	// LogDir is log directory of dfget.
101
	LogDir string `yaml:"logDir,omitempty" mapstructure:"logDir,omitempty"`
102

103
	// DataDir is data directory of dfget.
104
	DataDir string `yaml:"dataDir,omitempty" mapstructure:"dataDir,omitempty"`
105

106
	// CacheDir is cache directory of dfget.
107
	CacheDir string `yaml:"cacheDir,omitempty" mapstructure:"cacheDir,omitempty"`
108

109
	// WorkHome is working directory of dfget.
110
	WorkHome string `yaml:"workHome,omitempty" mapstructure:"workHome,omitempty"`
111

112
	RateLimit util.RateLimit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"`
113

114
	// Config file paths,
115
	// default:["/etc/dragonfly/dfget.yaml","/etc/dragonfly.conf"].
116
	//
117
	// NOTE: It is recommended to use `/etc/dragonfly/dfget.yaml` as default,
118
	// and the `/etc/dragonfly.conf` is just to ensure compatibility with previous versions.
119
	//ConfigFiles []string `json:"-"`
120

121
	// MoreDaemonOptions indicates more options passed to daemon by command line.
122
	MoreDaemonOptions string `yaml:"moreDaemonOptions,omitempty" mapstructure:"moreDaemonOptions,omitempty"`
123

124
	// Recursive indicates to download all resources in target url, the target source client must support list action
125
	Recursive bool `yaml:"recursive,omitempty" mapstructure:"recursive,omitempty"`
126

127
	// RecursiveList indicates to list all resources in target url, the target source client must support list action
128
	RecursiveList bool `yaml:"recursiveList,omitempty" mapstructure:"list,omitempty"`
129

130
	// RecursiveLevel indicates to the maximum number of subdirectories that dfget will recurse into
131
	RecursiveLevel uint `yaml:"recursiveLevel,omitempty" mapstructure:"level,omitempty"`
132

133
	RecursiveAcceptRegex string `yaml:"acceptRegex,omitempty" mapstructure:"accept-regex,omitempty"`
134

135
	RecursiveRejectRegex string `yaml:"rejectRegex,omitempty" mapstructure:"reject-regex,omitempty"`
136

137
	KeepOriginalOffset bool `yaml:"keepOriginalOffset,omitempty" mapstructure:"original-offset,omitempty"`
138

139
	// Range stands download range for url, like: 0-9, will download 10 bytes from 0 to 9 ([0:9])
140
	Range string `yaml:"range,omitempty" mapstructure:"range,omitempty"`
141
}
142

143
func NewDfgetConfig() *ClientOption {
144
	return &dfgetConfig
145
}
146

147
func (cfg *ClientOption) Validate() error {
148
	if cfg == nil {
149
		return fmt.Errorf("runtime config: %w", dferrors.ErrInvalidArgument)
150
	}
151

152
	if !url.IsValid(cfg.URL) {
153
		return fmt.Errorf("url %s: %w", cfg.URL, dferrors.ErrInvalidArgument)
154
	}
155

156
	if _, err := regexp.Compile(cfg.RecursiveAcceptRegex); err != nil {
157
		return err
158
	}
159

160
	if _, err := regexp.Compile(cfg.RecursiveRejectRegex); err != nil {
161
		return err
162
	}
163

164
	if err := cfg.checkOutput(); err != nil {
165
		return fmt.Errorf("output %s: %w", err.Error(), dferrors.ErrInvalidArgument)
166
	}
167

168
	if err := cfg.checkHeader(); err != nil {
169
		return fmt.Errorf("output %s: %w", err.Error(), dferrors.ErrInvalidHeader)
170
	}
171

172
	if int64(cfg.RateLimit.Limit) < DefaultMinRate.ToNumber() {
173
		return fmt.Errorf("rate limit must be greater than %s: %w", DefaultMinRate.String(), dferrors.ErrInvalidArgument)
174
	}
175

176
	return nil
177
}
178

179
func (cfg *ClientOption) Convert(args []string) error {
180
	if pkgstrings.IsBlank(cfg.Output) {
181
		url := strings.TrimRight(cfg.URL, "/")
182
		idx := strings.LastIndexByte(url, '/')
183
		if idx < 0 {
184
			return fmt.Errorf("get output from url[%s] error", cfg.URL)
185
		}
186
		cfg.Output = url[idx+1:]
187
	}
188

189
	if !filepath.IsAbs(cfg.Output) {
190
		absPath, err := filepath.Abs(cfg.Output)
191
		if err != nil {
192
			return fmt.Errorf("get absolute path[%s] error: %v", cfg.Output, err)
193
		}
194
		cfg.Output = absPath
195
	}
196
	if cfg.URL == "" && len(args) > 0 {
197
		cfg.URL = args[0]
198
	}
199

200
	if cfg.Digest != "" {
201
		cfg.Tag = ""
202
	}
203

204
	if cfg.Console {
205
		cfg.ShowProgress = false
206
	}
207
	return nil
208
}
209

210
func (cfg *ClientOption) String() string {
211
	data, _ := json.Marshal(cfg)
212
	return string(data)
213
}
214

215
// checkHeader is for checking the header format
216
func (cfg *ClientOption) checkHeader() error {
217
	if len(cfg.Header) == 0 {
218
		return nil
219
	}
220

221
	for _, header := range cfg.Header {
222
		if !strings.Contains(header, ":") {
223
			return fmt.Errorf("header format error: %v", header)
224
		}
225
		idx := strings.Index(header, ":")
226
		if len(strings.TrimSpace(header[:idx])) == 0 || len(strings.TrimSpace(header[idx+1:])) == 0 {
227
			return fmt.Errorf("header format error: %v", header)
228
		}
229
	}
230

231
	return nil
232
}
233

234
// This function must be called after checkURL
235
func (cfg *ClientOption) checkOutput() error {
236
	if !filepath.IsAbs(cfg.Output) {
237
		return fmt.Errorf("path[%s] is not absolute path", cfg.Output)
238
	}
239
	outputDir, _ := path.Split(cfg.Output)
240
	if err := MkdirAll(outputDir, 0700, os.Getuid(), os.Getgid()); err != nil {
241
		return err
242
	}
243

244
	f, err := os.Stat(cfg.Output)
245
	// when not recursive download, need a file
246
	if !cfg.Recursive && err == nil && f.IsDir() {
247
		return fmt.Errorf("path[%s] is directory but requires file path", cfg.Output)
248
	}
249

250
	// check permission
251
	for dir := cfg.Output; !pkgstrings.IsBlank(dir); dir = filepath.Dir(dir) {
252
		if err := syscall.Access(dir, syscall.O_RDWR); err == nil {
253
			break
254
		} else if os.IsPermission(err) || dir == "/" {
255
			return fmt.Errorf("user[%s] path[%s] %v", user.Username(), cfg.Output, err)
256
		}
257
	}
258
	return nil
259
}
260

261
// MkdirAll make directories recursive, and changes uid, gid to the latest directory.
262
// For example: the path /data/x exists, uid=1, gid=1
263
// when call MkdirAll("/data/x/y/z", 0755, 2, 2)
264
// MkdirAll creates /data/x/y and change owner to 2:2, creates /data/x/y/z and change owner to 2:2
265
func MkdirAll(dir string, perm os.FileMode, uid, gid int) error {
266
	var (
267
		// directories to create
268
		dirs []string
269
		err  error
270
	)
271
	subDir := dir
272
	// find not exist directories from bottom to top
273
	for {
274
		if subDir == "" {
275
			break
276
		}
277
		_, err = os.Stat(subDir)
278
		// directory exists
279
		if err == nil {
280
			break
281
		}
282
		if os.IsNotExist(err) {
283
			dirs = append(dirs, subDir)
284
		} else {
285
			logger.Errorf("stat error: %s", err)
286
			return err
287
		}
288
		subDir = path.Dir(subDir)
289
	}
290

291
	// no directory to create
292
	if len(dirs) == 0 {
293
		return nil
294
	}
295

296
	err = os.MkdirAll(dir, perm)
297
	if err != nil {
298
		logger.Errorf("mkdir error: %s", err)
299
		return err
300
	}
301

302
	// update owner from top to bottom
303
	for _, d := range dirs {
304
		err = os.Chown(d, uid, gid)
305
		if err != nil {
306
			logger.Errorf("chown error: %s", err)
307
			return err
308
		}
309
	}
310
	return nil
311
}
312