cubefs

1
// Copyright 2022 The CubeFS Authors.
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
// implied. See the License for the specific language governing
13
// permissions and limitations under the License.
14

15
package main
16

17
import (
18
	"flag"
19
	"fmt"
20
	syslog "log"
21
	"net/http"
22
	"net/http/pprof"
23
	"os"
24
	"os/signal"
25
	"path"
26
	"path/filepath"
27
	"runtime"
28
	"strings"
29
	"syscall"
30

31
	"github.com/cubefs/cubefs/blockcache/bcache"
32
	"github.com/cubefs/cubefs/cmd/common"
33
	"github.com/cubefs/cubefs/proto"
34
	"github.com/cubefs/cubefs/util/config"
35
	"github.com/cubefs/cubefs/util/errors"
36
	"github.com/cubefs/cubefs/util/log"
37
	"github.com/cubefs/cubefs/util/stat"
38
	sysutil "github.com/cubefs/cubefs/util/sys"
39
	"github.com/cubefs/cubefs/util/ump"
40
	"github.com/jacobsa/daemonize"
41
)
42

43
//TODO: remove this later.
44
//go:generate golangci-lint run --issues-exit-code=1 -D errcheck -E bodyclose ./...
45

46
const (
47
	ConfigKeyLogDir     = "logDir"
48
	ConfigKeyLogLevel   = "logLevel"
49
	ConfigKeyProfPort   = "prof"
50
	ConfigKeyWarnLogDir = "warnLogDir"
51

52
	RoleBcache = "blockcache"
53

54
	LoggerOutput = "output.log"
55
)
56

57
var (
58
	configFile       = flag.String("c", "", "config file path")
59
	configVersion    = flag.Bool("v", false, "show version")
60
	configForeground = flag.Bool("f", false, "run foreground")
61
)
62

63
func interceptSignal(s common.Server) {
64
	sigC := make(chan os.Signal, 1)
65
	signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)
66
	syslog.Println("action[interceptSignal] register system signal.")
67
	go func() {
68
		for {
69
			sig := <-sigC
70
			syslog.Printf("action[interceptSignal] received signal: %s. pid %d", sig.String(), os.Getpid())
71
			s.Shutdown()
72
		}
73
	}()
74
}
75

76
func modifyOpenFiles() (err error) {
77
	var rLimit syscall.Rlimit
78
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
79
	if err != nil {
80
		return fmt.Errorf("Error Getting Rlimit %v", err.Error())
81
	}
82
	syslog.Println(rLimit)
83
	rLimit.Max = 1024000
84
	rLimit.Cur = 1024000
85
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
86
	if err != nil {
87
		return fmt.Errorf("Error Setting Rlimit %v", err.Error())
88
	}
89
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
90
	if err != nil {
91
		return fmt.Errorf("Error Getting Rlimit %v", err.Error())
92
	}
93
	syslog.Println("Rlimit Final", rLimit)
94
	return
95
}
96

97
func main() {
98
	if os.Args[1] == "stop" {
99
		os.Exit(0)
100
	}
101

102
	flag.Parse()
103

104
	Version := proto.DumpVersion("Server")
105
	if *configVersion {
106
		fmt.Printf("%v", Version)
107
		os.Exit(0)
108
	}
109

110
	/*
111
	 * LoadConfigFile should be checked before start daemon, since it will
112
	 * call os.Exit() w/o notifying the parent process.
113
	 */
114
	cfg, err := config.LoadConfigFile(*configFile)
115
	if err != nil {
116
		daemonize.SignalOutcome(err)
117
		os.Exit(1)
118
	}
119

120
	if !*configForeground {
121
		if err := startDaemon(); err != nil {
122
			fmt.Printf("Server start failed: %v\n", err)
123
			os.Exit(1)
124
		}
125
		os.Exit(0)
126
	}
127

128
	/*
129
	 * We are in daemon from here.
130
	 * Must notify the parent process through SignalOutcome anyway.
131
	 */
132

133
	role := RoleBcache
134
	logDir := cfg.GetString(ConfigKeyLogDir)
135
	logLevel := cfg.GetString(ConfigKeyLogLevel)
136
	profPort := cfg.GetString(ConfigKeyProfPort)
137
	umpDatadir := cfg.GetString(ConfigKeyWarnLogDir)
138

139
	// Init server instance with specified role configuration.
140
	var (
141
		server common.Server
142
		module string
143
	)
144
	switch role {
145
	case RoleBcache:
146
		server = bcache.NewServer()
147
		module = RoleBcache
148
	default:
149
		err = errors.NewErrorf("Fatal: role mismatch: %s", role)
150
		fmt.Println(err)
151
		daemonize.SignalOutcome(err)
152
		os.Exit(1)
153
	}
154

155
	// Init logging
156
	var (
157
		level log.Level
158
	)
159
	switch strings.ToLower(logLevel) {
160
	case "debug":
161
		level = log.DebugLevel
162
	case "info":
163
		level = log.InfoLevel
164
	case "warn":
165
		level = log.WarnLevel
166
	case "error":
167
		level = log.ErrorLevel
168
	default:
169
		level = log.ErrorLevel
170
	}
171

172
	_, err = log.InitLog(logDir, module, level, nil, log.DefaultLogLeftSpaceLimit)
173
	if err != nil {
174
		err = errors.NewErrorf("Fatal: failed to init log - %v", err)
175
		fmt.Println(err)
176
		daemonize.SignalOutcome(err)
177
		os.Exit(1)
178
	}
179
	defer log.LogFlush()
180

181
	// Init output file
182
	outputFilePath := path.Join(logDir, module, LoggerOutput)
183
	outputFile, err := os.OpenFile(outputFilePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o666)
184
	if err != nil {
185
		err = errors.NewErrorf("Fatal: failed to open output path - %v", err)
186
		fmt.Println(err)
187
		daemonize.SignalOutcome(err)
188
		os.Exit(1)
189
	}
190
	// stat log
191
	_, err = stat.NewStatistic(logDir, "blockcache", int64(stat.DefaultStatLogSize),
192
		stat.DefaultTimeOutUs, true)
193
	if err != nil {
194
		err = errors.NewErrorf("Init stat log fail: %v\n", err)
195
		fmt.Println(err)
196
		daemonize.SignalOutcome(err)
197
		os.Exit(1)
198
	}
199
	stat.ClearStat()
200

201
	defer func() {
202
		outputFile.Sync()
203
		outputFile.Close()
204
	}()
205
	syslog.SetOutput(outputFile)
206

207
	if err = sysutil.RedirectFD(int(outputFile.Fd()), int(os.Stderr.Fd())); err != nil {
208
		err = errors.NewErrorf("Fatal: failed to redirect fd - %v", err)
209
		syslog.Println(err)
210
		daemonize.SignalOutcome(err)
211
		os.Exit(1)
212
	}
213

214
	syslog.Printf("Hello, CubeFS Storage\n%s\n", Version)
215

216
	err = modifyOpenFiles()
217
	if err != nil {
218
		err = errors.NewErrorf("Fatal: failed to modify open files - %v", err)
219
		syslog.Println(err)
220
		daemonize.SignalOutcome(err)
221
		os.Exit(1)
222
	}
223

224
	// for multi-cpu scheduling
225
	runtime.GOMAXPROCS(runtime.NumCPU())
226
	if err = ump.InitUmp(role, umpDatadir); err != nil {
227
		log.LogFlush()
228
		err = errors.NewErrorf("Fatal: failed to init ump warnLogDir - %v", err)
229
		syslog.Println(err)
230
		daemonize.SignalOutcome(err)
231
		os.Exit(1)
232
	}
233

234
	if profPort != "" {
235
		go func() {
236
			mainMux := http.NewServeMux()
237
			mux := http.NewServeMux()
238
			http.HandleFunc(log.SetLogLevelPath, log.SetLogLevel)
239
			mux.Handle("/debug/pprof", http.HandlerFunc(pprof.Index))
240
			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
241
			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
242
			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
243
			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
244
			mux.Handle("/debug/", http.HandlerFunc(pprof.Index))
245
			mainHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
246
				if strings.HasPrefix(req.URL.Path, "/debug/") {
247
					mux.ServeHTTP(w, req)
248
				} else {
249
					http.DefaultServeMux.ServeHTTP(w, req)
250
				}
251
			})
252
			mainMux.Handle("/", mainHandler)
253
			e := http.ListenAndServe(fmt.Sprintf(":%v", profPort), mainMux)
254
			if e != nil {
255
				log.LogFlush()
256
				err = errors.NewErrorf("cannot listen pprof %v err %v", profPort, err)
257
				syslog.Println(err)
258
				daemonize.SignalOutcome(err)
259
				os.Exit(1)
260
			}
261
		}()
262
	}
263

264
	interceptSignal(server)
265
	err = server.Start(cfg)
266
	if err != nil {
267
		log.LogFlush()
268
		err = errors.NewErrorf("Fatal: failed to start the CubeFS %s daemon err %v - ", role, err)
269
		syslog.Println(err)
270
		daemonize.SignalOutcome(err)
271
		os.Exit(1)
272
	}
273

274
	daemonize.SignalOutcome(nil)
275

276
	// Block main goroutine until server shutdown.
277
	server.Sync()
278
	log.LogFlush()
279
	os.Exit(0)
280
}
281

282
func startDaemon() error {
283
	cmdPath, err := os.Executable()
284
	if err != nil {
285
		return fmt.Errorf("startDaemon failed: cannot get absolute command path, err(%v)", err)
286
	}
287

288
	configPath, err := filepath.Abs(*configFile)
289
	if err != nil {
290
		return fmt.Errorf("startDaemon failed: cannot get absolute command path of config file(%v) , err(%v)", *configFile, err)
291
	}
292

293
	args := []string{"-f"}
294
	args = append(args, "-c")
295
	args = append(args, configPath)
296

297
	env := []string{
298
		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
299
	}
300

301
	err = daemonize.Run(cmdPath, args, env, os.Stdout)
302
	if err != nil {
303
		return fmt.Errorf("startDaemon failed: daemon start failed, cmd(%v) args(%v) env(%v) err(%v)\n", cmdPath, args, env, err)
304
	}
305

306
	return nil
307
}
308