Dragonfly2
711 строк · 19.6 Кб
1/*
2* Copyright 2020 The Dragonfly Authors
3*
4* Licensed under the Apache License, Version 2.0 (the "License");
5* you may not use this file except in compliance with the License.
6* You may obtain a copy of the License at
7*
8* http://www.apache.org/licenses/LICENSE-2.0
9*
10* Unless required by applicable law or agreed to in writing, software
11* distributed under the License is distributed on an "AS IS" BASIS,
12* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13* See the License for the specific language governing permissions and
14* limitations under the License.
15*/
16
17package config18
19import (20"errors"21"fmt"22"net"23"time"24
25"d7y.io/dragonfly/v2/cmd/dependency/base"26"d7y.io/dragonfly/v2/pkg/net/ip"27"d7y.io/dragonfly/v2/pkg/objectstorage"28"d7y.io/dragonfly/v2/pkg/rpc"29"d7y.io/dragonfly/v2/pkg/slices"30"d7y.io/dragonfly/v2/pkg/types"31)
32
33type Config struct {34// Base options.35base.Options `yaml:",inline" mapstructure:",squash"`36
37// Server configuration.38Server ServerConfig `yaml:"server" mapstructure:"server"`39
40// Auth configuration.41Auth AuthConfig `yaml:"auth" mapstructure:"auth"`42
43// Database configuration.44Database DatabaseConfig `yaml:"database" mapstructure:"database"`45
46// Cache configuration.47Cache CacheConfig `yaml:"cache" mapstructure:"cache"`48
49// Job configuration.50Job JobConfig `yaml:"job" mapstructure:"job"`51
52// ObjectStorage configuration.53ObjectStorage ObjectStorageConfig `yaml:"objectStorage" mapstructure:"objectStorage"`54
55// Metrics configuration.56Metrics MetricsConfig `yaml:"metrics" mapstructure:"metrics"`57
58// Security configuration.59Security SecurityConfig `yaml:"security" mapstructure:"security"`60
61// Network configuration.62Network NetworkConfig `yaml:"network" mapstructure:"network"`63
64// Trainer configuration.65Trainer TrainerConfig `yaml:"trainer" mapstructure:"trainer"`66}
67
68type ServerConfig struct {69// Server name.70Name string `yaml:"name" mapstructure:"name"`71
72// Server work directory.73WorkHome string `yaml:"workHome" mapstructure:"workHome"`74
75// Server dynamic config cache directory.76CacheDir string `yaml:"cacheDir" mapstructure:"cacheDir"`77
78// Server log directory.79LogDir string `yaml:"logDir" mapstructure:"logDir"`80
81// Server plugin directory.82PluginDir string `yaml:"pluginDir" mapstructure:"pluginDir"`83
84// GRPC server configuration.85GRPC GRPCConfig `yaml:"grpc" mapstructure:"grpc"`86
87// REST server configuration.88REST RESTConfig `yaml:"rest" mapstructure:"rest"`89}
90
91type AuthConfig struct {92// JWT configuration.93JWT JWTConfig `yaml:"jwt" mapstructure:"jwt"`94}
95
96type JWTConfig struct {97// Realm name to display to the user, default value is Dragonfly.98Realm string `yaml:"realm" mapstructure:"realm"`99
100// Key is secret key used for signing. Please change the key in production101Key string `yaml:"key" mapstructure:"key"`102
103// Timeout is duration that a jwt token is valid, default duration is two days.104Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"`105
106// MaxRefresh field allows clients to refresh their token until MaxRefresh has passed, default duration is two days.107MaxRefresh time.Duration `yaml:"maxRefresh" mapstructure:"maxRefresh"`108}
109
110type DatabaseConfig struct {111// Database type.112Type string `yaml:"type" mapstructure:"type"`113
114// Mysql configuration.115Mysql MysqlConfig `yaml:"mysql" mapstructure:"mysql"`116
117// Postgres configuration.118Postgres PostgresConfig `yaml:"postgres" mapstructure:"postgres"`119
120// Redis configuration.121Redis RedisConfig `yaml:"redis" mapstructure:"redis"`122}
123
124type MysqlConfig struct {125// Server username.126User string `yaml:"user" mapstructure:"user"`127
128// Server password.129Password string `yaml:"password" mapstructure:"password"`130
131// Server host.132Host string `yaml:"host" mapstructure:"host"`133
134// Server port.135Port int `yaml:"port" mapstructure:"port"`136
137// Server DB name.138DBName string `yaml:"dbname" mapstructure:"dbname"`139
140// TLS mode (can be one of "true", "false", "skip-verify", or "preferred").141TLSConfig string `yaml:"tlsConfig" mapstructure:"tlsConfig"`142
143// Custom TLS client configuration (overrides "TLSConfig" setting above).144TLS *MysqlTLSClientConfig `yaml:"tls" mapstructure:"tls"`145
146// Enable migration.147Migrate bool `yaml:"migrate" mapstructure:"migrate"`148}
149
150type MysqlTLSClientConfig struct {151// Client certificate file path.152Cert string `yaml:"cert" mapstructure:"cert"`153
154// Client key file path.155Key string `yaml:"key" mapstructure:"key"`156
157// CA file path.158CA string `yaml:"ca" mapstructure:"ca"`159
160// InsecureSkipVerify controls whether a client verifies the161// server's certificate chain and host name.162InsecureSkipVerify bool `yaml:"insecureSkipVerify" mapstructure:"insecureSkipVerify"`163}
164
165type PostgresConfig struct {166// Server username.167User string `yaml:"user" mapstructure:"user"`168
169// Server password.170Password string `yaml:"password" mapstructure:"password"`171
172// Server host.173Host string `yaml:"host" mapstructure:"host"`174
175// Server port.176Port int `yaml:"port" mapstructure:"port"`177
178// Server DB name.179DBName string `yaml:"dbname" mapstructure:"dbname"`180
181// SSL mode.182SSLMode string `yaml:"sslMode" mapstructure:"sslMode"`183
184// Disable prepared statement.185PreferSimpleProtocol bool `yaml:"preferSimpleProtocol" mapstructure:"preferSimpleProtocol"`186
187// Server timezone.188Timezone string `yaml:"timezone" mapstructure:"timezone"`189
190// Enable migration.191Migrate bool `yaml:"migrate" mapstructure:"migrate"`192}
193
194type RedisConfig struct {195// DEPRECATED: Please use the `addrs` field instead.196Host string `yaml:"host" mapstructure:"host"`197
198// DEPRECATED: Please use the `addrs` field instead.199Port int `yaml:"port" mapstructure:"port"`200
201// Addrs is server addresses.202Addrs []string `yaml:"addrs" mapstructure:"addrs"`203
204// MasterName is the sentinel master name.205MasterName string `yaml:"masterName" mapstructure:"masterName"`206
207// Username is server username.208Username string `yaml:"username" mapstructure:"username"`209
210// Password is server password.211Password string `yaml:"password" mapstructure:"password"`212
213// DB is server cache DB name.214DB int `yaml:"db" mapstructure:"db"`215
216// BrokerDB is server broker DB name.217BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"`218
219// BackendDB is server backend DB name.220BackendDB int `yaml:"backendDB" mapstructure:"backendDB"`221}
222
223type CacheConfig struct {224// Redis cache configuration.225Redis RedisCacheConfig `yaml:"redis" mapstructure:"redis"`226
227// Local cache configuration.228Local LocalCacheConfig `yaml:"local" mapstructure:"local"`229}
230
231type RedisCacheConfig struct {232// Cache TTL.233TTL time.Duration `yaml:"ttl" mapstructure:"ttl"`234}
235
236type LocalCacheConfig struct {237// Size of LFU cache.238Size int `yaml:"size" mapstructure:"size"`239
240// Cache TTL.241TTL time.Duration `yaml:"ttl" mapstructure:"ttl"`242}
243
244type RESTConfig struct {245// REST server address.246Addr string `yaml:"addr" mapstructure:"addr"`247
248// TLS server configuration.249TLS *TLSServerConfig `yaml:"tls" mapstructure:"tls"`250}
251
252type TLSServerConfig struct {253// Certificate file path.254Cert string `yaml:"cert" mapstructure:"cert"`255
256// Key file path.257Key string `yaml:"key" mapstructure:"key"`258}
259
260type MetricsConfig struct {261// Enable metrics service.262Enable bool `yaml:"enable" mapstructure:"enable"`263
264// Metrics service address.265Addr string `yaml:"addr" mapstructure:"addr"`266}
267
268type GRPCConfig struct {269// AdvertiseIP is advertise ip.270AdvertiseIP net.IP `yaml:"advertiseIP" mapstructure:"advertiseIP"`271
272// ListenIP is listen ip, like: 0.0.0.0, 192.168.0.1.273ListenIP net.IP `mapstructure:"listenIP" yaml:"listenIP"`274
275// Port is listen port.276PortRange TCPListenPortRange `yaml:"port" mapstructure:"port"`277}
278
279type TCPListenPortRange struct {280Start int281End int282}
283
284type JobConfig struct {285// Preheat configuration.286Preheat PreheatConfig `yaml:"preheat" mapstructure:"preheat"`287
288// Sync peers configuration.289SyncPeers SyncPeersConfig `yaml:"syncPeers" mapstructure:"syncPeers"`290}
291
292type PreheatConfig struct {293// RegistryTimeout is the timeout for requesting registry to get token and manifest.294RegistryTimeout time.Duration `yaml:"registryTimeout" mapstructure:"registryTimeout"`295
296// TLS client configuration.297TLS *PreheatTLSClientConfig `yaml:"tls" mapstructure:"tls"`298}
299
300type SyncPeersConfig struct {301// Interval is the interval for syncing all peers information from the scheduler and302// display peers information in the manager console.303Interval time.Duration `yaml:"interval" mapstructure:"interval"`304
305// Timeout is the timeout for syncing peers information from the single scheduler.306Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"`307}
308
309type PreheatTLSClientConfig struct {310// CACert is the CA certificate for preheat tls handshake, it can be path or PEM format string.311CACert types.PEMContent `yaml:"caCert" mapstructure:"caCert"`312}
313
314type ObjectStorageConfig struct {315// Enable object storage.316Enable bool `yaml:"enable" mapstructure:"enable"`317
318// Name is object storage name of type, it can be s3, oss or obs.319Name string `mapstructure:"name" yaml:"name"`320
321// Region is storage region.322Region string `mapstructure:"region" yaml:"region"`323
324// Endpoint is datacenter endpoint.325Endpoint string `mapstructure:"endpoint" yaml:"endpoint"`326
327// AccessKey is access key ID.328AccessKey string `mapstructure:"accessKey" yaml:"accessKey"`329
330// SecretKey is access key secret.331SecretKey string `mapstructure:"secretKey" yaml:"secretKey"`332
333// S3ForcePathStyle sets force path style for s3, true by default.334// Set this to `true` to force the request to use path-style addressing,335// i.e., `http://s3.amazonaws.com/BUCKET/KEY`. By default, the S3 client336// will use virtual hosted bucket addressing when possible337// (`http://BUCKET.s3.amazonaws.com/KEY`).338// Refer to https://github.com/aws/aws-sdk-go/blob/main/aws/config.go#L118.339S3ForcePathStyle bool `mapstructure:"s3ForcePathStyle" yaml:"s3ForcePathStyle"`340}
341
342type SecurityConfig struct {343// AutoIssueCert indicates to issue client certificates for all grpc call.344AutoIssueCert bool `yaml:"autoIssueCert" mapstructure:"autoIssueCert"`345
346// CACert is the CA certificate for all grpc tls handshake, it can be path or PEM format string.347CACert types.PEMContent `mapstructure:"caCert" yaml:"caCert"`348
349// CAKey is the CA private key, it can be path or PEM format string.350CAKey types.PEMContent `mapstructure:"caKey" yaml:"caKey"`351
352// TLSPolicy controls the grpc shandshake behaviors:353// force: both ClientHandshake and ServerHandshake are only support tls354// prefer: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support tls355// default: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support insecure (non-tls)356TLSPolicy string `mapstructure:"tlsPolicy" yaml:"tlsPolicy"`357
358// CertSpec is the desired state of certificate.359CertSpec CertSpec `mapstructure:"certSpec" yaml:"certSpec"`360}
361
362type CertSpec struct {363// DNSNames is a list of dns names be set on the certificate.364DNSNames []string `mapstructure:"dnsNames" yaml:"dnsNames"`365
366// IPAddresses is a list of ip addresses be set on the certificate.367IPAddresses []net.IP `mapstructure:"ipAddresses" yaml:"ipAddresses"`368
369// ValidityPeriod is the validity period of certificate.370ValidityPeriod time.Duration `mapstructure:"validityPeriod" yaml:"validityPeriod"`371}
372
373type NetworkConfig struct {374// EnableIPv6 enables ipv6 for server.375EnableIPv6 bool `mapstructure:"enableIPv6" yaml:"enableIPv6"`376}
377
378type TrainerConfig struct {379// Enable trainer service.380Enable bool `yaml:"enable" mapstructure:"enable"`381
382// BucketName is the object storage bucket name of model.383BucketName string `yaml:"bucketName" mapstructure:"bucketName"`384}
385
386// New config instance.
387func New() *Config {388return &Config{389Server: ServerConfig{390Name: DefaultServerName,391GRPC: GRPCConfig{392PortRange: TCPListenPortRange{393Start: DefaultGRPCPort,394End: DefaultGRPCPort,395},396},397REST: RESTConfig{398Addr: DefaultRESTAddr,399},400},401Auth: AuthConfig{402JWT: JWTConfig{403Realm: DefaultJWTRealm,404Timeout: DefaultJWTTimeout,405MaxRefresh: DefaultJWTMaxRefresh,406},407},408Database: DatabaseConfig{409Type: DatabaseTypeMysql,410Mysql: MysqlConfig{411Port: DefaultMysqlPort,412DBName: DefaultMysqlDBName,413Migrate: true,414},415Postgres: PostgresConfig{416Port: DefaultPostgresPort,417DBName: DefaultPostgresDBName,418SSLMode: DefaultPostgresSSLMode,419PreferSimpleProtocol: DefaultPostgresPreferSimpleProtocol,420Timezone: DefaultPostgresTimezone,421Migrate: true,422},423Redis: RedisConfig{424DB: DefaultRedisDB,425BrokerDB: DefaultRedisBrokerDB,426BackendDB: DefaultRedisBackendDB,427},428},429Cache: CacheConfig{430Redis: RedisCacheConfig{431TTL: DefaultRedisCacheTTL,432},433Local: LocalCacheConfig{434Size: DefaultLFUCacheSize,435TTL: DefaultLFUCacheTTL,436},437},438Job: JobConfig{439Preheat: PreheatConfig{440RegistryTimeout: DefaultJobPreheatRegistryTimeout,441},442SyncPeers: SyncPeersConfig{443Interval: DefaultJobSyncPeersInterval,444Timeout: DefaultJobSyncPeersTimeout,445},446},447ObjectStorage: ObjectStorageConfig{448Enable: false,449S3ForcePathStyle: true,450},451Security: SecurityConfig{452AutoIssueCert: false,453TLSPolicy: rpc.PreferTLSPolicy,454CertSpec: CertSpec{455DNSNames: DefaultCertDNSNames,456IPAddresses: DefaultCertIPAddresses,457ValidityPeriod: DefaultCertValidityPeriod,458},459},460Metrics: MetricsConfig{461Enable: false,462Addr: DefaultMetricsAddr,463},464Network: NetworkConfig{465EnableIPv6: DefaultNetworkEnableIPv6,466},467Trainer: TrainerConfig{468Enable: false,469BucketName: DefaultTrainerBucketName,470},471}472}
473
474// Validate config values
475func (cfg *Config) Validate() error {476if cfg.Server.Name == "" {477return errors.New("server requires parameter name")478}479
480if cfg.Server.GRPC.AdvertiseIP == nil {481return errors.New("grpc requires parameter advertiseIP")482}483
484if cfg.Server.GRPC.ListenIP == nil {485return errors.New("grpc requires parameter listenIP")486}487
488if cfg.Server.REST.TLS != nil {489if cfg.Server.REST.TLS.Cert == "" {490return errors.New("tls requires parameter cert")491}492
493if cfg.Server.REST.TLS.Key == "" {494return errors.New("tls requires parameter key")495}496}497
498if cfg.Auth.JWT.Realm == "" {499return errors.New("jwt requires parameter realm")500}501
502if cfg.Auth.JWT.Key == "" {503return errors.New("jwt requires parameter key")504}505
506if cfg.Auth.JWT.Timeout == 0 {507return errors.New("jwt requires parameter timeout")508}509
510if cfg.Auth.JWT.MaxRefresh == 0 {511return errors.New("jwt requires parameter maxRefresh")512}513
514if cfg.Database.Type == "" {515return errors.New("database requires parameter type")516}517
518if slices.Contains([]string{DatabaseTypeMysql, DatabaseTypeMariaDB}, cfg.Database.Type) {519if cfg.Database.Mysql.User == "" {520return errors.New("mysql requires parameter user")521}522
523if cfg.Database.Mysql.Password == "" {524return errors.New("mysql requires parameter password")525}526
527if cfg.Database.Mysql.Host == "" {528return errors.New("mysql requires parameter host")529}530
531if cfg.Database.Mysql.Port <= 0 {532return errors.New("mysql requires parameter port")533}534
535if cfg.Database.Mysql.DBName == "" {536return errors.New("mysql requires parameter dbname")537}538
539if cfg.Database.Mysql.TLS != nil {540if cfg.Database.Mysql.TLS.Cert == "" {541return errors.New("tls requires parameter cert")542}543
544if cfg.Database.Mysql.TLS.Key == "" {545return errors.New("tls requires parameter key")546}547
548if cfg.Database.Mysql.TLS.CA == "" {549return errors.New("tls requires parameter ca")550}551}552}553
554if cfg.Database.Type == DatabaseTypePostgres {555if cfg.Database.Postgres.User == "" {556return errors.New("postgres requires parameter user")557}558
559if cfg.Database.Postgres.Password == "" {560return errors.New("postgres requires parameter password")561}562
563if cfg.Database.Postgres.Host == "" {564return errors.New("postgres requires parameter host")565}566
567if cfg.Database.Postgres.Port <= 0 {568return errors.New("postgres requires parameter port")569}570
571if cfg.Database.Postgres.DBName == "" {572return errors.New("postgres requires parameter dbname")573}574
575if cfg.Database.Postgres.SSLMode == "" {576return errors.New("postgres requires parameter sslMode")577}578
579if cfg.Database.Postgres.Timezone == "" {580return errors.New("postgres requires parameter timezone")581}582}583
584if len(cfg.Database.Redis.Addrs) == 0 {585return errors.New("redis requires parameter addrs")586}587
588if cfg.Database.Redis.DB < 0 {589return errors.New("redis requires parameter db")590}591
592if cfg.Database.Redis.BrokerDB < 0 {593return errors.New("redis requires parameter brokerDB")594}595
596if cfg.Database.Redis.BackendDB < 0 {597return errors.New("redis requires parameter backendDB")598}599
600if cfg.Cache.Redis.TTL == 0 {601return errors.New("redis requires parameter ttl")602}603
604if cfg.Cache.Local.Size == 0 {605return errors.New("local requires parameter size")606}607
608if cfg.Cache.Local.TTL == 0 {609return errors.New("local requires parameter ttl")610}611
612if cfg.Job.Preheat.TLS != nil {613if cfg.Job.Preheat.TLS.CACert == "" {614return errors.New("preheat requires parameter caCert")615}616}617
618if cfg.Job.Preheat.RegistryTimeout == 0 {619return errors.New("preheat requires parameter registryTimeout")620}621
622if cfg.Job.SyncPeers.Interval <= MinJobSyncPeersInterval {623return errors.New("syncPeers requires parameter interval and it must be greater than 12 hours")624}625
626if cfg.Job.SyncPeers.Timeout == 0 {627return errors.New("syncPeers requires parameter timeout")628}629
630if cfg.ObjectStorage.Enable {631if cfg.ObjectStorage.Name == "" {632return errors.New("objectStorage requires parameter name")633}634
635if !slices.Contains([]string{objectstorage.ServiceNameS3, objectstorage.ServiceNameOSS, objectstorage.ServiceNameOBS}, cfg.ObjectStorage.Name) {636return errors.New("objectStorage requires parameter name")637}638
639if cfg.ObjectStorage.AccessKey == "" {640return errors.New("objectStorage requires parameter accessKey")641}642
643if cfg.ObjectStorage.SecretKey == "" {644return errors.New("objectStorage requires parameter secretKey")645}646}647
648if cfg.Metrics.Enable {649if cfg.Metrics.Addr == "" {650return errors.New("metrics requires parameter addr")651}652}653
654if cfg.Security.AutoIssueCert {655if cfg.Security.CACert == "" {656return errors.New("security requires parameter caCert")657}658
659if cfg.Security.CAKey == "" {660return errors.New("security requires parameter caKey")661}662
663if !slices.Contains([]string{rpc.DefaultTLSPolicy, rpc.ForceTLSPolicy, rpc.PreferTLSPolicy}, cfg.Security.TLSPolicy) {664return errors.New("security requires parameter tlsPolicy")665}666
667if len(cfg.Security.CertSpec.IPAddresses) == 0 {668return errors.New("certSpec requires parameter ipAddresses")669}670
671if len(cfg.Security.CertSpec.DNSNames) == 0 {672return errors.New("certSpec requires parameter dnsNames")673}674
675if cfg.Security.CertSpec.ValidityPeriod <= 0 {676return errors.New("certSpec requires parameter validityPeriod")677}678}679
680if cfg.Trainer.Enable {681if cfg.Trainer.BucketName == "" {682return errors.New("trainer requires parameter bucketName")683}684}685return nil686}
687
688func (cfg *Config) Convert() error {689// TODO Compatible with deprecated fields host and port.690if len(cfg.Database.Redis.Addrs) == 0 && cfg.Database.Redis.Host != "" && cfg.Database.Redis.Port > 0 {691cfg.Database.Redis.Addrs = []string{fmt.Sprintf("%s:%d", cfg.Database.Redis.Host, cfg.Database.Redis.Port)}692}693
694if cfg.Server.GRPC.AdvertiseIP == nil {695if cfg.Network.EnableIPv6 {696cfg.Server.GRPC.AdvertiseIP = ip.IPv6697} else {698cfg.Server.GRPC.AdvertiseIP = ip.IPv4699}700}701
702if cfg.Server.GRPC.ListenIP == nil {703if cfg.Network.EnableIPv6 {704cfg.Server.GRPC.ListenIP = net.IPv6zero705} else {706cfg.Server.GRPC.ListenIP = net.IPv4zero707}708}709
710return nil711}
712