cubefs

1
// Copyright 2018 The CubeFS Authors.
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
// implied. See the License for the specific language governing
13
// permissions and limitations under the License.
14

15
package main
16

17
import (
18
	"flag"
19
	"fmt"
20
	syslog "log"
21
	"net/http"
22
	"net/http/pprof"
23
	"os"
24
	"os/signal"
25
	"path"
26
	"path/filepath"
27
	"runtime"
28
	"strconv"
29
	"strings"
30
	"syscall"
31

32
	"github.com/cubefs/cubefs/authnode"
33
	"github.com/cubefs/cubefs/cmd/common"
34
	"github.com/cubefs/cubefs/console"
35
	"github.com/cubefs/cubefs/datanode"
36
	"github.com/cubefs/cubefs/lcnode"
37
	"github.com/cubefs/cubefs/master"
38
	"github.com/cubefs/cubefs/metanode"
39
	"github.com/cubefs/cubefs/objectnode"
40
	"github.com/cubefs/cubefs/proto"
41
	"github.com/cubefs/cubefs/util/auditlog"
42
	"github.com/cubefs/cubefs/util/config"
43
	"github.com/cubefs/cubefs/util/errors"
44
	"github.com/cubefs/cubefs/util/log"
45
	sysutil "github.com/cubefs/cubefs/util/sys"
46
	"github.com/cubefs/cubefs/util/ump"
47
	"github.com/jacobsa/daemonize"
48
)
49

50
const (
51
	ConfigKeyRole              = "role"
52
	ConfigKeyLogDir            = "logDir"
53
	ConfigKeyLogLevel          = "logLevel"
54
	ConfigKeyLogRotateSize     = "logRotateSize"
55
	ConfigKeyLogRotateHeadRoom = "logRotateHeadRoom"
56
	ConfigKeyProfPort          = "prof"
57
	ConfigKeyWarnLogDir        = "warnLogDir"
58
	ConfigKeyBuffersTotalLimit = "buffersTotalLimit"
59
	ConfigKeyLogLeftSpaceLimit = "logLeftSpaceLimit"
60
)
61

62
const (
63
	RoleMaster    = "master"
64
	RoleMeta      = "metanode"
65
	RoleData      = "datanode"
66
	RoleAuth      = "authnode"
67
	RoleObject    = "objectnode"
68
	RoleConsole   = "console"
69
	RoleLifeCycle = "lcnode"
70
)
71

72
const (
73
	ModuleMaster    = "master"
74
	ModuleMeta      = "metaNode"
75
	ModuleData      = "dataNode"
76
	ModuleAuth      = "authNode"
77
	ModuleObject    = "objectNode"
78
	ModuleConsole   = "console"
79
	ModuleLifeCycle = "lcnode"
80
)
81

82
const (
83
	LoggerOutput = "output.log"
84
)
85

86
var (
87
	configFile       = flag.String("c", "", "config file path")
88
	configVersion    = flag.Bool("v", false, "show version")
89
	configForeground = flag.Bool("f", false, "run foreground")
90
	redirectSTD      = flag.Bool("redirect-std", true, "redirect standard output to file")
91
)
92

93
func interceptSignal(s common.Server) {
94
	sigC := make(chan os.Signal, 1)
95
	signal.Notify(sigC, syscall.SIGINT, syscall.SIGTERM)
96
	syslog.Println("action[interceptSignal] register system signal.")
97
	go func() {
98
		for {
99
			sig := <-sigC
100
			syslog.Printf("action[interceptSignal] received signal: %s. pid %d", sig.String(), os.Getpid())
101
			s.Shutdown()
102
		}
103
	}()
104
}
105

106
func modifyOpenFiles() (err error) {
107
	var rLimit syscall.Rlimit
108
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
109
	if err != nil {
110
		return fmt.Errorf("Error Getting Rlimit %v", err.Error())
111
	}
112
	syslog.Println(rLimit)
113
	rLimit.Max = 1024000
114
	rLimit.Cur = 1024000
115
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
116
	if err != nil {
117
		return fmt.Errorf("Error Setting Rlimit %v", err.Error())
118
	}
119
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
120
	if err != nil {
121
		return fmt.Errorf("Error Getting Rlimit %v", err.Error())
122
	}
123
	syslog.Println("Rlimit Final", rLimit)
124
	return
125
}
126

127
func main() {
128
	flag.Parse()
129

130
	Version := proto.DumpVersion("Server")
131
	if *configVersion {
132
		fmt.Printf("%v", Version)
133
		os.Exit(0)
134
	}
135

136
	/*
137
	 * LoadConfigFile should be checked before start daemon, since it will
138
	 * call os.Exit() w/o notifying the parent process.
139
	 */
140
	cfg, err := config.LoadConfigFile(*configFile)
141
	if err != nil {
142
		daemonize.SignalOutcome(err)
143
		os.Exit(1)
144
	}
145

146
	if !*configForeground {
147
		if err := startDaemon(); err != nil {
148
			fmt.Printf("Server start failed: %v\n", err)
149
			os.Exit(1)
150
		}
151
		os.Exit(0)
152
	}
153

154
	/*
155
	 * We are in daemon from here.
156
	 * Must notify the parent process through SignalOutcome anyway.
157
	 */
158

159
	role := cfg.GetString(ConfigKeyRole)
160
	logDir := cfg.GetString(ConfigKeyLogDir)
161
	logLevel := cfg.GetString(ConfigKeyLogLevel)
162
	logRotateSize := cfg.GetInt64(ConfigKeyLogRotateSize)
163
	logRotateHeadRoom := cfg.GetInt64(ConfigKeyLogRotateHeadRoom)
164
	profPort := cfg.GetString(ConfigKeyProfPort)
165
	umpDatadir := cfg.GetString(ConfigKeyWarnLogDir)
166
	buffersTotalLimit := cfg.GetInt64(ConfigKeyBuffersTotalLimit)
167
	logLeftSpaceLimitStr := cfg.GetString(ConfigKeyLogLeftSpaceLimit)
168
	logLeftSpaceLimit, err := strconv.ParseInt(logLeftSpaceLimitStr, 10, 64)
169
	if err != nil || logLeftSpaceLimit == 0 {
170
		log.LogErrorf("logLeftSpaceLimit is not a legal int value: %v", err.Error())
171
		logLeftSpaceLimit = log.DefaultLogLeftSpaceLimit
172
	}
173
	// Init server instance with specified role configuration.
174
	var (
175
		server common.Server
176
		module string
177
	)
178
	switch role {
179
	case RoleMeta:
180
		server = metanode.NewServer()
181
		module = ModuleMeta
182
	case RoleMaster:
183
		server = master.NewServer()
184
		module = ModuleMaster
185
	case RoleData:
186
		server = datanode.NewServer()
187
		module = ModuleData
188
	case RoleAuth:
189
		server = authnode.NewServer()
190
		module = ModuleAuth
191
	case RoleObject:
192
		server = objectnode.NewServer()
193
		module = ModuleObject
194
	case RoleConsole:
195
		server = console.NewServer()
196
		module = ModuleConsole
197
	case RoleLifeCycle:
198
		server = lcnode.NewServer()
199
		module = ModuleLifeCycle
200
	default:
201
		err = errors.NewErrorf("Fatal: role mismatch: %s", role)
202
		fmt.Println(err)
203
		daemonize.SignalOutcome(err)
204
		os.Exit(1)
205
	}
206

207
	// Init logging
208
	var (
209
		level log.Level
210
	)
211
	switch strings.ToLower(logLevel) {
212
	case "debug":
213
		level = log.DebugLevel
214
	case "info":
215
		level = log.InfoLevel
216
	case "warn":
217
		level = log.WarnLevel
218
	case "error":
219
		level = log.ErrorLevel
220
	case "critical":
221
		level = log.CriticalLevel
222
	default:
223
		level = log.ErrorLevel
224
	}
225
	rotate := log.NewLogRotate()
226
	if logRotateSize > 0 {
227
		rotate.SetRotateSizeMb(logRotateSize)
228
	}
229
	if logRotateHeadRoom > 0 {
230
		rotate.SetHeadRoomMb(logRotateHeadRoom)
231
	}
232
	_, err = log.InitLog(logDir, module, level, rotate, logLeftSpaceLimit)
233
	if err != nil {
234
		err = errors.NewErrorf("Fatal: failed to init log - %v", err)
235
		fmt.Println(err)
236
		daemonize.SignalOutcome(err)
237
		os.Exit(1)
238
	}
239
	defer log.LogFlush()
240
	if errors.SupportPanicHook() {
241
		err = errors.AtPanic(func() {
242
			log.LogFlush()
243
		})
244
		if err != nil {
245
			log.LogErrorf("failed to hook go panic")
246
			err = nil
247
		}
248
	}
249

250
	_, err = auditlog.InitAudit(logDir, module, auditlog.DefaultAuditLogSize)
251
	if err != nil {
252
		err = errors.NewErrorf("Fatal: failed to init audit log - %v", err)
253
		fmt.Println(err)
254
		daemonize.SignalOutcome(err)
255
		os.Exit(1)
256
	}
257
	defer auditlog.StopAudit()
258

259
	if *redirectSTD {
260
		// Init output file
261
		outputFilePath := path.Join(logDir, module, LoggerOutput)
262
		outputFile, err := os.OpenFile(outputFilePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o666)
263
		if err != nil {
264
			err = errors.NewErrorf("Fatal: failed to open output path - %v", err)
265
			fmt.Println(err)
266
			daemonize.SignalOutcome(err)
267
			os.Exit(1)
268
		}
269
		defer func() {
270
			outputFile.Sync()
271
			outputFile.Close()
272
		}()
273

274
		syslog.SetOutput(outputFile)
275
		if err = sysutil.RedirectFD(int(outputFile.Fd()), int(os.Stderr.Fd())); err != nil {
276
			err = errors.NewErrorf("Fatal: failed to redirect fd - %v", err)
277
			syslog.Println(err)
278
			daemonize.SignalOutcome(err)
279
			os.Exit(1)
280
		}
281
	}
282

283
	if buffersTotalLimit < 0 {
284
		syslog.Printf("invalid fields, BuffersTotalLimit(%v) must larger or equal than 0\n", buffersTotalLimit)
285
		return
286
	}
287

288
	proto.InitBufferPool(buffersTotalLimit)
289
	syslog.Printf("Hello, CubeFS Storage\n%s\n", Version)
290
	err = modifyOpenFiles()
291
	if err != nil {
292
		err = errors.NewErrorf("Fatal: failed to modify open files - %v", err)
293
		syslog.Println(err)
294
		daemonize.SignalOutcome(err)
295
		os.Exit(1)
296
	}
297

298
	// for multi-cpu scheduling
299
	runtime.GOMAXPROCS(runtime.NumCPU())
300
	if err = ump.InitUmp(role, umpDatadir); err != nil {
301
		log.LogFlush()
302
		err = errors.NewErrorf("Fatal: failed to init ump warnLogDir - %v", err)
303
		syslog.Println(err)
304
		daemonize.SignalOutcome(err)
305
		os.Exit(1)
306
	}
307

308
	if profPort != "" {
309
		go func() {
310
			mainMux := http.NewServeMux()
311
			mux := http.NewServeMux()
312
			http.HandleFunc(log.SetLogLevelPath, log.SetLogLevel)
313
			mux.Handle("/debug/pprof", http.HandlerFunc(pprof.Index))
314
			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
315
			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
316
			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
317
			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
318
			mux.Handle("/debug/", http.HandlerFunc(pprof.Index))
319
			mainHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
320
				if strings.HasPrefix(req.URL.Path, "/debug/") {
321
					mux.ServeHTTP(w, req)
322
				} else {
323
					http.DefaultServeMux.ServeHTTP(w, req)
324
				}
325
			})
326
			mainMux.Handle("/", mainHandler)
327
			e := http.ListenAndServe(fmt.Sprintf(":%v", profPort), mainMux)
328
			if e != nil {
329
				log.LogFlush()
330
				err = errors.NewErrorf("cannot listen pprof %v err %v", profPort, e)
331
				syslog.Println(err)
332
				daemonize.SignalOutcome(err)
333
				os.Exit(1)
334
			}
335
		}()
336
	}
337

338
	interceptSignal(server)
339
	err = server.Start(cfg)
340
	if err != nil {
341
		log.LogFlush()
342
		err = errors.NewErrorf("Fatal: failed to start the CubeFS %s daemon err %v - ", role, err)
343
		syslog.Println(err)
344
		daemonize.SignalOutcome(err)
345
		os.Exit(1)
346
	}
347

348
	syslog.Printf("server start success, pid %d, role %s", os.Getpid(), role)
349
	log.LogDisableStderrOutput()
350
	err = log.OutputPid(logDir, role)
351
	if err != nil {
352
		log.LogFlush()
353
		err = errors.NewErrorf("Fatal: failed to print pid %s err %v - ", role, err)
354
		syslog.Println(err)
355
		daemonize.SignalOutcome(err)
356
		os.Exit(1)
357
	}
358

359
	daemonize.SignalOutcome(nil)
360

361
	// Block main goroutine until server shutdown.
362
	server.Sync()
363
	log.LogFlush()
364
	os.Exit(0)
365
}
366

367
func startDaemon() error {
368
	cmdPath, err := os.Executable()
369
	if err != nil {
370
		return fmt.Errorf("startDaemon failed: cannot get absolute command path, err(%v)", err)
371
	}
372

373
	configPath, err := filepath.Abs(*configFile)
374
	if err != nil {
375
		return fmt.Errorf("startDaemon failed: cannot get absolute command path of config file(%v) , err(%v)", *configFile, err)
376
	}
377

378
	args := []string{"-f"}
379
	args = append(args, "-c")
380
	args = append(args, configPath)
381

382
	env := []string{
383
		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
384
	}
385

386
	err = daemonize.Run(cmdPath, args, env, os.Stdout)
387
	if err != nil {
388
		return fmt.Errorf("startDaemon failed: daemon start failed, cmd(%v) args(%v) env(%v) err(%v)\n", cmdPath, args, env, err)
389
	}
390

391
	return nil
392
}
393