Dragonfly2
789 строк · 17.8 Кб
1/*
2* Copyright 2020 The Dragonfly Authors
3*
4* Licensed under the Apache License, Version 2.0 (the "License");
5* you may not use this file except in compliance with the License.
6* You may obtain a copy of the License at
7*
8* http://www.apache.org/licenses/LICENSE-2.0
9*
10* Unless required by applicable law or agreed to in writing, software
11* distributed under the License is distributed on an "AS IS" BASIS,
12* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13* See the License for the specific language governing permissions and
14* limitations under the License.
15*/
16
17package config
18
19import (
20"fmt"
21"net"
22"net/url"
23"os"
24"reflect"
25"strings"
26"testing"
27"time"
28
29"github.com/stretchr/testify/assert"
30"golang.org/x/time/rate"
31"gopkg.in/yaml.v3"
32
33"d7y.io/dragonfly/v2/client/util"
34"d7y.io/dragonfly/v2/cmd/dependency/base"
35"d7y.io/dragonfly/v2/pkg/dfnet"
36"d7y.io/dragonfly/v2/pkg/types"
37"d7y.io/dragonfly/v2/pkg/unit"
38)
39
40func Test_AllUnmarshalYAML(t *testing.T) {
41var cases = []struct {
42text string
43target any
44}{
45{
46text: `
47"port": 1234
48`,
49target: &struct {
50Port TCPListenPortRange `yaml:"port"`
51}{
52Port: TCPListenPortRange{
53Start: 1234,
54},
55},
56},
57{
58text: `
59port:
60start: 1234
61end: 1235
62`,
63target: &struct {
64Port TCPListenPortRange `yaml:"port"`
65}{
66Port: TCPListenPortRange{
67Start: 1234,
68End: 1235,
69},
70},
71},
72{
73text: `
74timeout: 1000000000
75`,
76target: &struct {
77Timeout util.Duration `yaml:"timeout"`
78}{
79Timeout: util.Duration{
80Duration: time.Second,
81},
82},
83},
84{
85text: `
86timeout: 1s
87`,
88target: &struct {
89Timeout util.Duration `yaml:"timeout"`
90}{
91Timeout: util.Duration{
92Duration: time.Second,
93},
94},
95},
96{
97text: `
98limit: 100Mi
99`,
100target: &struct {
101Limit util.RateLimit `yaml:"limit"`
102}{
103Limit: util.RateLimit{
104Limit: 100 * 1024 * 1024,
105},
106},
107},
108{
109text: `
110limit: 2097152
111`,
112target: &struct {
113Limit util.RateLimit `yaml:"limit"`
114}{
115Limit: util.RateLimit{
116Limit: 2 * 1024 * 1024,
117},
118},
119},
120{
121text: `
122addr: 127.0.0.1:8002
123`,
124target: &struct {
125Addr dfnet.NetAddr `yaml:"addr"`
126}{
127Addr: dfnet.NetAddr{
128Type: dfnet.TCP,
129Addr: "127.0.0.1:8002",
130},
131},
132},
133{
134text: `
135listen:
136type: tcp
137addr: 127.0.0.1:8002
138`,
139target: &struct {
140Listen dfnet.NetAddr `yaml:"listen"`
141}{
142Listen: dfnet.NetAddr{
143Type: dfnet.TCP,
144Addr: "127.0.0.1:8002",
145},
146},
147},
148{
149text: `
150diskGCThreshold: 1Ki
151`,
152target: &struct {
153Size unit.Bytes `yaml:"diskGCThreshold"`
154}{
155Size: unit.Bytes(1024),
156},
157},
158}
159for _, c := range cases {
160actual := reflect.New(reflect.TypeOf(c.target).Elem()).Interface()
161err := yaml.Unmarshal([]byte(c.text), actual)
162
163assert := assert.New(t)
164assert.Nil(err, "yaml.Unmarshal should return nil")
165assert.EqualValues(c.target, actual)
166}
167}
168
169func TestUnmarshalYAML(t *testing.T) {
170bytes := []byte(`
171tls:
172key: ./testdata/certs/sca.key
173cert: ./testdata/certs/sca.crt
174caCert: ./testdata/certs/ca.crt
175url: https://d7y.io
176certs: ["./testdata/certs/ca.crt", "./testdata/certs/sca.crt"]
177regx: blobs/sha256.*
178port1: 1001
179port2:
180start: 1002
181end: 1003
182timeout: 3m
183limit: 2Mib
184type: tcp
185proxy1: ./testdata/config/proxy.yaml
186proxy2:
187registryMirror:
188url: https://index.docker.io
189schedulers1:
190netAddrs:
191- 0.0.0.0
192- 0.0.0.1
193scheduleTimeout: 0
194schedulers2:
195netAddrs:
196- type: tcp
197addr: 0.0.0.0
198scheduleTimeout: 0
199`)
200
201var s = struct {
202TLSConfig *TLSConfig `yaml:"tls"`
203URL *URL `yaml:"url"`
204Certs *CertPool `yaml:"certs"`
205Regx *Regexp `yaml:"regx"`
206Port1 TCPListenPortRange `yaml:"port1"`
207Port2 TCPListenPortRange `yaml:"port2"`
208Timeout util.Duration `yaml:"timeout"`
209Limit util.RateLimit `yaml:"limit"`
210Type dfnet.NetworkType `yaml:"type"`
211Proxy1 ProxyOption `yaml:"proxy1"`
212Proxy2 ProxyOption `yaml:"proxy2"`
213Schedulers1 SchedulerOption `yaml:"schedulers1"`
214Schedulers2 SchedulerOption `yaml:"schedulers2"`
215}{}
216
217if err := yaml.Unmarshal(bytes, &s); err != nil {
218t.Fatal(err)
219}
220}
221
222func TestPeerHostOption_Load(t *testing.T) {
223proxyExp, _ := NewRegexp("blobs/sha256.*")
224hijackExp, _ := NewRegexp("mirror.aliyuncs.com:443")
225
226_caCert, _ := os.ReadFile("./testdata/certs/ca.crt")
227_cert, _ := os.ReadFile("./testdata/certs/sca.crt")
228_key, _ := os.ReadFile("./testdata/certs/sca.key")
229
230caCert := types.PEMContent(strings.TrimSpace(string(_caCert)))
231cert := types.PEMContent(strings.TrimSpace(string(_cert)))
232key := types.PEMContent(strings.TrimSpace(string(_key)))
233
234peerHostOption := &DaemonOption{
235Options: base.Options{
236Console: true,
237Verbose: true,
238PProfPort: -1,
239Telemetry: base.TelemetryOption{
240Jaeger: "foo",
241ServiceName: "bar",
242},
243},
244AliveTime: util.Duration{
245Duration: 0,
246},
247GCInterval: util.Duration{
248Duration: 60000000000,
249},
250Metrics: ":8000",
251WorkHome: "/tmp/dragonfly/dfdaemon/",
252WorkHomeMode: 0700,
253CacheDir: "/var/cache/dragonfly/",
254CacheDirMode: 0700,
255LogDir: "/var/log/dragonfly/",
256PluginDir: "/tmp/dragonfly/dfdaemon/plugins/",
257DataDir: "/var/lib/dragonfly/",
258DataDirMode: 0700,
259KeepStorage: false,
260Scheduler: SchedulerOption{
261Manager: ManagerOption{
262Enable: false,
263NetAddrs: []dfnet.NetAddr{
264{
265Type: dfnet.TCP,
266Addr: "127.0.0.1:65003",
267},
268},
269RefreshInterval: 5 * time.Minute,
270SeedPeer: SeedPeerOption{
271Enable: false,
272Type: types.HostTypeStrongSeedName,
273ClusterID: 2,
274KeepAlive: KeepAliveOption{
275Interval: 10 * time.Second,
276},
277},
278},
279NetAddrs: []dfnet.NetAddr{
280{
281Type: dfnet.TCP,
282Addr: "127.0.0.1:8002",
283},
284},
285ScheduleTimeout: util.Duration{
286Duration: 0,
287},
288DisableAutoBackSource: true,
289},
290Host: HostOption{
291Hostname: "d7y.io",
292Location: "0.0.0.0",
293IDC: "d7y",
294AdvertiseIP: net.IPv4zero,
295},
296Download: DownloadOption{
297TotalRateLimit: util.RateLimit{
298Limit: 1024 * 1024 * 1024,
299},
300PerPeerRateLimit: util.RateLimit{
301Limit: 512 * 1024 * 1024,
302},
303PieceDownloadTimeout: 30 * time.Second,
304DownloadGRPC: ListenOption{
305Security: SecurityOption{
306Insecure: true,
307CACert: caCert,
308Cert: cert,
309Key: key,
310TLSVerify: true,
311TLSConfig: nil,
312},
313TCPListen: nil,
314UnixListen: &UnixListenOption{
315Socket: "/tmp/dfdaemon.sock",
316},
317},
318PeerGRPC: ListenOption{
319Security: SecurityOption{
320Insecure: true,
321CACert: caCert,
322Cert: cert,
323Key: key,
324TLSVerify: true,
325TLSConfig: nil,
326},
327TCPListen: &TCPListenOption{
328Listen: "0.0.0.0",
329PortRange: TCPListenPortRange{
330Start: 65000,
331End: 0,
332},
333},
334},
335CalculateDigest: true,
336Transport: &TransportOption{
337DialTimeout: time.Second,
338KeepAlive: time.Second,
339MaxIdleConns: 1,
340IdleConnTimeout: time.Second,
341ResponseHeaderTimeout: time.Second,
342TLSHandshakeTimeout: time.Second,
343ExpectContinueTimeout: time.Second,
344},
345GetPiecesMaxRetry: 1,
346Prefetch: true,
347WatchdogTimeout: time.Second,
348Concurrent: &ConcurrentOption{
349ThresholdSize: util.Size{
350Limit: 1,
351},
352ThresholdSpeed: unit.Bytes(1),
353GoroutineCount: 1,
354InitBackoff: 1,
355MaxBackoff: 1,
356MaxAttempts: 1,
357},
358},
359Upload: UploadOption{
360RateLimit: util.RateLimit{
361Limit: 1024 * 1024 * 1024,
362},
363ListenOption: ListenOption{
364Security: SecurityOption{
365Insecure: true,
366CACert: caCert,
367Cert: cert,
368Key: key,
369TLSVerify: true,
370},
371TCPListen: &TCPListenOption{
372Listen: "0.0.0.0",
373PortRange: TCPListenPortRange{
374Start: 65002,
375End: 0,
376},
377},
378},
379},
380ObjectStorage: ObjectStorageOption{
381Enable: true,
382Filter: "Expires&Signature&ns",
383MaxReplicas: 3,
384ListenOption: ListenOption{
385Security: SecurityOption{
386Insecure: true,
387CACert: caCert,
388Cert: cert,
389Key: key,
390TLSVerify: true,
391},
392TCPListen: &TCPListenOption{
393Listen: "0.0.0.0",
394PortRange: TCPListenPortRange{
395Start: 65004,
396End: 0,
397},
398},
399},
400},
401Storage: StorageOption{
402DataPath: "/tmp/storage/data",
403TaskExpireTime: util.Duration{
404Duration: 180000000000,
405},
406StoreStrategy: StoreStrategy("io.d7y.storage.v2.simple"),
407DiskGCThreshold: 60 * unit.MB,
408DiskGCThresholdPercent: 0.6,
409Multiplex: true,
410},
411Health: &HealthOption{
412Path: "/health",
413},
414Proxy: &ProxyOption{
415ListenOption: ListenOption{
416Security: SecurityOption{
417Insecure: true,
418CACert: caCert,
419Cert: cert,
420Key: key,
421TLSVerify: true,
422},
423TCPListen: &TCPListenOption{
424Listen: "0.0.0.0",
425PortRange: TCPListenPortRange{
426Start: 65001,
427End: 0,
428},
429},
430},
431BasicAuth: &BasicAuth{
432Username: "foo",
433Password: "bar",
434},
435DefaultFilter: "baz",
436DefaultTag: "tag",
437DefaultApplication: "application",
438MaxConcurrency: 1,
439RegistryMirror: &RegistryMirror{
440Remote: &URL{
441&url.URL{
442Host: "index.docker.io",
443Scheme: "https",
444},
445},
446DynamicRemote: true,
447UseProxies: true,
448Insecure: true,
449Direct: false,
450},
451WhiteList: []*WhiteList{
452{
453Host: "foo",
454Regx: proxyExp,
455Ports: []string{
456"1000",
457"2000",
458},
459},
460},
461ProxyRules: []*ProxyRule{
462{
463Regx: proxyExp,
464UseHTTPS: false,
465Direct: false,
466Redirect: "d7y.io",
467},
468},
469HijackHTTPS: &HijackConfig{
470Cert: "./testdata/certs/sca.crt",
471Key: "./testdata/certs/sca.key",
472Hosts: []*HijackHost{
473{
474Regx: hijackExp,
475Insecure: true,
476},
477},
478SNI: nil,
479},
480DumpHTTPContent: true,
481ExtraRegistryMirrors: []*RegistryMirror{
482{
483Remote: &URL{
484&url.URL{
485Host: "index.docker.io",
486Scheme: "https",
487},
488},
489DynamicRemote: true,
490UseProxies: true,
491Insecure: true,
492Direct: true,
493},
494},
495},
496Reload: ReloadOption{
497Interval: util.Duration{
498Duration: 180000000000,
499},
500},
501Security: GlobalSecurityOption{
502AutoIssueCert: true,
503CACert: "-----BEGIN CERTIFICATE-----",
504TLSVerify: true,
505TLSPolicy: "force",
506CertSpec: &CertSpec{
507DNSNames: []string{"foo"},
508IPAddresses: []net.IP{net.IPv4zero},
509ValidityPeriod: 1000000000,
510},
511},
512Network: &NetworkOption{
513EnableIPv6: true,
514},
515Announcer: AnnouncerOption{
516SchedulerInterval: 1000000000,
517},
518NetworkTopology: NetworkTopologyOption{
519Enable: true,
520Probe: ProbeOption{
521Interval: 20 * time.Minute,
522},
523},
524}
525
526peerHostOptionYAML := &DaemonOption{}
527if err := peerHostOptionYAML.Load("./testdata/config/daemon.yaml"); err != nil {
528t.Fatal(err)
529}
530
531assert := assert.New(t)
532assert.EqualValues(peerHostOption, peerHostOptionYAML)
533}
534
535func TestPeerHostOption_Validate(t *testing.T) {
536tests := []struct {
537name string
538config *DaemonConfig
539mock func(cfg *DaemonConfig)
540expect func(t *testing.T, err error)
541}{
542{
543name: "valid config",
544config: NewDaemonConfig(),
545mock: func(cfg *DaemonConfig) {
546cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
547{
548Type: dfnet.TCP,
549Addr: "127.0.0.1:8002",
550},
551}
552},
553expect: func(t *testing.T, err error) {
554assert := assert.New(t)
555assert.NoError(err)
556},
557},
558{
559name: "manager addr is not specified",
560config: NewDaemonConfig(),
561mock: func(cfg *DaemonConfig) {
562cfg.Scheduler.Manager.Enable = true
563cfg.Scheduler.Manager.NetAddrs = nil
564},
565expect: func(t *testing.T, err error) {
566assert := assert.New(t)
567assert.EqualError(err, "manager addr is not specified")
568},
569},
570{
571name: "manager refreshInterval not specified",
572config: NewDaemonConfig(),
573mock: func(cfg *DaemonConfig) {
574cfg.Scheduler.Manager.Enable = true
575cfg.Scheduler.Manager.RefreshInterval = 0
576cfg.Scheduler.Manager.NetAddrs = []dfnet.NetAddr{
577{
578Type: dfnet.TCP,
579Addr: "127.0.0.1:8002",
580},
581}
582},
583expect: func(t *testing.T, err error) {
584assert := assert.New(t)
585assert.EqualError(err, "manager refreshInterval is not specified")
586},
587},
588{
589name: "empty schedulers and config server is not specified",
590config: NewDaemonConfig(),
591mock: func(cfg *DaemonConfig) {
592cfg.Scheduler.NetAddrs = nil
593},
594expect: func(t *testing.T, err error) {
595assert := assert.New(t)
596assert.EqualError(err, "empty schedulers and config server is not specified")
597},
598},
599{
600name: "download rate limit must be greater",
601config: NewDaemonConfig(),
602mock: func(cfg *DaemonConfig) {
603cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
604{
605Type: dfnet.TCP,
606Addr: "127.0.0.1:8002",
607},
608}
609cfg.Download.TotalRateLimit.Limit = rate.Limit(10 * unit.MB)
610},
611expect: func(t *testing.T, err error) {
612assert := assert.New(t)
613msg := fmt.Sprintf("rate limit must be greater than %s", DefaultMinRate.String())
614assert.EqualError(err, msg)
615},
616},
617{
618name: "upload rate limit must be greater",
619config: NewDaemonConfig(),
620mock: func(cfg *DaemonConfig) {
621cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
622{
623Type: dfnet.TCP,
624Addr: "127.0.0.1:8002",
625},
626}
627cfg.Upload.RateLimit.Limit = rate.Limit(10 * unit.MB)
628},
629expect: func(t *testing.T, err error) {
630assert := assert.New(t)
631msg := fmt.Sprintf("rate limit must be greater than %s", DefaultMinRate.String())
632assert.EqualError(err, msg)
633},
634},
635{
636name: "max replicas must be greater than 0",
637config: NewDaemonConfig(),
638mock: func(cfg *DaemonConfig) {
639cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
640{
641Type: dfnet.TCP,
642Addr: "127.0.0.1:8002",
643},
644}
645cfg.ObjectStorage.Enable = true
646cfg.ObjectStorage.MaxReplicas = 0
647},
648expect: func(t *testing.T, err error) {
649assert := assert.New(t)
650assert.EqualError(err, "max replicas must be greater than 0")
651},
652},
653{
654name: "reload interval too short, must great than 1 second",
655config: NewDaemonConfig(),
656mock: func(cfg *DaemonConfig) {
657cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
658{
659Type: dfnet.TCP,
660Addr: "127.0.0.1:8002",
661},
662}
663cfg.Reload.Interval.Duration = time.Millisecond
664},
665expect: func(t *testing.T, err error) {
666assert := assert.New(t)
667assert.EqualError(err, "reload interval too short, must great than 1 second")
668},
669},
670{
671name: "gcInterval must be greater than 0",
672config: NewDaemonConfig(),
673mock: func(cfg *DaemonConfig) {
674cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
675{
676Type: dfnet.TCP,
677Addr: "127.0.0.1:8002",
678},
679}
680cfg.GCInterval.Duration = 0
681},
682expect: func(t *testing.T, err error) {
683assert := assert.New(t)
684assert.EqualError(err, "gcInterval must be greater than 0")
685},
686},
687{
688name: "security requires parameter caCert",
689config: NewDaemonConfig(),
690mock: func(cfg *DaemonConfig) {
691cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
692{
693Type: dfnet.TCP,
694Addr: "127.0.0.1:8002",
695},
696}
697cfg.Security.AutoIssueCert = true
698cfg.Security.CACert = ""
699},
700expect: func(t *testing.T, err error) {
701assert := assert.New(t)
702assert.EqualError(err, "security requires parameter caCert")
703},
704},
705{
706name: "certSpec requires parameter ipAddresses",
707config: NewDaemonConfig(),
708mock: func(cfg *DaemonConfig) {
709cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
710{
711Type: dfnet.TCP,
712Addr: "127.0.0.1:8002",
713},
714}
715cfg.Security.AutoIssueCert = true
716cfg.Security.CACert = "test"
717cfg.Security.CertSpec.IPAddresses = nil
718},
719expect: func(t *testing.T, err error) {
720assert := assert.New(t)
721assert.EqualError(err, "certSpec requires parameter ipAddresses")
722},
723},
724{
725name: "certSpec requires parameter dnsNames",
726config: NewDaemonConfig(),
727mock: func(cfg *DaemonConfig) {
728cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
729{
730Type: dfnet.TCP,
731Addr: "127.0.0.1:8002",
732},
733}
734cfg.Security.AutoIssueCert = true
735cfg.Security.CACert = "test"
736cfg.Security.CertSpec.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")}
737cfg.Security.CertSpec.DNSNames = nil
738},
739expect: func(t *testing.T, err error) {
740assert := assert.New(t)
741assert.EqualError(err, "certSpec requires parameter dnsNames")
742},
743},
744{
745name: "certSpec requires parameter validityPeriod",
746config: NewDaemonConfig(),
747mock: func(cfg *DaemonConfig) {
748cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
749{
750Type: dfnet.TCP,
751Addr: "127.0.0.1:8002",
752},
753}
754cfg.Security.AutoIssueCert = true
755cfg.Security.CACert = "testcert"
756cfg.Security.CertSpec.ValidityPeriod = 0
757},
758expect: func(t *testing.T, err error) {
759assert := assert.New(t)
760assert.EqualError(err, "certSpec requires parameter validityPeriod")
761},
762},
763{
764name: "probe requires parameter interval",
765config: NewDaemonConfig(),
766mock: func(cfg *DaemonConfig) {
767cfg.Scheduler.NetAddrs = []dfnet.NetAddr{
768{
769Type: dfnet.TCP,
770Addr: "127.0.0.1:8002",
771},
772}
773cfg.NetworkTopology.Enable = true
774cfg.NetworkTopology.Probe.Interval = 0
775},
776expect: func(t *testing.T, err error) {
777assert := assert.New(t)
778assert.EqualError(err, "probe requires parameter interval")
779},
780},
781}
782
783for _, tc := range tests {
784t.Run(tc.name, func(t *testing.T) {
785tc.mock(tc.config)
786tc.expect(t, tc.config.Validate())
787})
788}
789}
790