moira

1
package main
2

3
import (
4
	"flag"
5
	"fmt"
6
	"math/rand"
7
	"os"
8
	"os/signal"
9
	"syscall"
10
	"time"
11

12
	"go.avito.ru/DO/moira"
13
	"go.avito.ru/DO/moira/cmd"
14
	"go.avito.ru/DO/moira/database/neo4j"
15
	"go.avito.ru/DO/moira/database/redis"
16
	"go.avito.ru/DO/moira/fan"
17
	"go.avito.ru/DO/moira/logging"
18
	"go.avito.ru/DO/moira/metrics"
19
	"go.avito.ru/DO/moira/notifier"
20
	"go.avito.ru/DO/moira/notifier/escalations"
21
	"go.avito.ru/DO/moira/notifier/events"
22
	"go.avito.ru/DO/moira/notifier/notifications"
23
	"go.avito.ru/DO/moira/notifier/selfstate"
24
	"go.avito.ru/DO/moira/panicwrap"
25
	"go.avito.ru/DO/moira/sentry"
26
)
27

28
const (
29
	serviceName = "notifier"
30
)
31

32
var (
33
	logger                 moira.Logger
34
	configFileName         = flag.String("config", "/etc/moira/notifier.yml", "path to config file")
35
	printVersion           = flag.Bool("version", false, "Print current version and exit")
36
	printDefaultConfigFlag = flag.Bool("default-config", false, "Print default config and exit")
37
)
38

39
// Moira notifier bin version
40
var (
41
	MoiraVersion = "unknown"
42
	GitCommit    = "unknown"
43
	GoVersion    = "unknown"
44
)
45

46
func main() {
47
	rand.Seed(time.Now().UnixNano())
48

49
	flag.Parse()
50
	if *printVersion {
51
		fmt.Println("Moira Notifier")
52
		fmt.Println("Version:", MoiraVersion)
53
		fmt.Println("Git Commit:", GitCommit)
54
		fmt.Println("Go Version:", GoVersion)
55
		os.Exit(0)
56
	}
57

58
	config := getDefault()
59
	if *printDefaultConfigFlag {
60
		cmd.PrintConfig(config)
61
		os.Exit(0)
62
	}
63

64
	err := cmd.ReadConfig(*configFileName, &config)
65
	if err != nil {
66
		_, _ = fmt.Fprintf(os.Stderr, "Can not read settings: %s\n", err.Error())
67
		os.Exit(1)
68
	}
69

70
	if err = metrics.Init(config.Statsd.GetSettings(), config.Notifier.LimitMetrics.GetSettings()); err != nil {
71
		_, _ = fmt.Fprintf(os.Stderr, "Can not configure metrics: %v\n", err)
72
		os.Exit(1)
73
	}
74

75
	if err = logging.Init(logging.ComponentNotifier, config.Rsyslog.GetSettings(), config.Notifier.LimitLogger.GetSettings()); err != nil {
76
		_, _ = fmt.Fprintf(os.Stderr, "Can not configure log: %v\n", err)
77
		os.Exit(1)
78
	}
79

80
	if err = sentry.Init(config.Notifier.Sentry.GetSettings(), serviceName); err != nil {
81
		_, _ = fmt.Fprintf(os.Stderr, "Can not configure sentry: %v\n", err)
82
		os.Exit(1)
83
	}
84
	panicwrap.Init(serviceName)
85

86
	logger = logging.GetLogger("")
87
	defer logger.InfoF("Moira Notifier Stopped. Version: %s", MoiraVersion)
88

89
	if config.Pprof.Listen != "" {
90
		logger.InfoF("Starting pprof server at: [%s]", config.Pprof.Listen)
91
		cmd.StartProfiling(logger, config.Pprof)
92
	}
93

94
	if config.Liveness.Listen != "" {
95
		logger.InfoF("Starting liveness server at: [%s]", config.Liveness.Listen)
96
		cmd.StartLiveness(logger, config.Liveness)
97
	}
98

99
	databaseSettings := config.Redis.GetSettings()
100
	database := redis.NewDatabase(logger, databaseSettings)
101
	notifierMetrics := metrics.NewNotifierMetrics()
102

103
	triggerInheritanceDatabase, err := neo4j.NewDatabase(logger, config.Neo4j)
104
	if err != nil {
105
		logger.FatalF("Can not configure Neo4j: %v", err)
106
	}
107

108
	notifierConfig := config.Notifier.getSettings(logger)
109
	sender := notifier.NewNotifier(database, notifierConfig, notifierMetrics)
110

111
	// Register moira senders
112
	if err := sender.RegisterSenders(database); err != nil {
113
		logger.FatalF("Can not configure senders: %v", err)
114
	}
115
	defer database.DeregisterBots()
116

117
	// Start moira self state checker
118
	selfState := &selfstate.SelfCheckWorker{
119
		Config:   config.Notifier.SelfState.getSettings(),
120
		DB:       database,
121
		Logger:   logger,
122
		Notifier: sender,
123
	}
124
	if err := selfState.Start(); err != nil {
125
		logger.FatalF("SelfState failed: %v", err)
126
	}
127
	defer stopSelfStateChecker(selfState)
128

129
	// Start moira notification fetcher
130
	fetchNotificationsWorker := &notifications.FetchNotificationsWorker{
131
		Logger:                     logger,
132
		Database:                   database,
133
		TriggerInheritanceDatabase: triggerInheritanceDatabase,
134
		Notifier:                   sender,
135
	}
136
	fetchNotificationsWorker.Start()
137
	defer stopNotificationsFetcher(fetchNotificationsWorker)
138

139
	// Start moira new events fetchers
140
	for _, withSaturations := range [...]bool{true, false} {
141
		// bring `withSaturations` into this scope
142
		// without this line, both workers would have `withSaturations == false`
143
		withSaturations := withSaturations
144

145
		fetchEventsWorker := &events.FetchEventsWorker{
146
			Logger:                     logger,
147
			Database:                   database,
148
			TriggerInheritanceDatabase: triggerInheritanceDatabase,
149
			Scheduler:                  notifier.NewScheduler(database, notifierMetrics),
150
			Metrics:                    notifierMetrics,
151
			Fan:                        fan.NewClient(config.Notifier.FanURL),
152

153
			Fetcher: func() (events moira.NotificationEvents, err error) {
154
				event, err := database.FetchNotificationEvent(withSaturations)
155
				return moira.NotificationEvents{event}, err
156
			},
157
		}
158
		fetchEventsWorker.Start()
159
		defer stopFetchEvents(fetchEventsWorker)
160
	}
161

162
	// Start moira delayed events fetchers
163
	for _, withSaturations := range [...]bool{true, false} {
164
		withSaturations := withSaturations
165
		fetchDelayedEventsWorker := &events.FetchEventsWorker{
166
			Logger:                     logger,
167
			Database:                   database,
168
			TriggerInheritanceDatabase: triggerInheritanceDatabase,
169
			Scheduler:                  notifier.NewScheduler(database, notifierMetrics),
170
			Metrics:                    notifierMetrics,
171
			Fan:                        fan.NewClient(config.Notifier.FanURL),
172

173
			Fetcher: func() (events moira.NotificationEvents, err error) {
174
				time.Sleep(5 * time.Second)
175
				return database.FetchDelayedNotificationEvents(time.Now().Unix(), withSaturations)
176
			},
177
		}
178
		fetchDelayedEventsWorker.Start()
179
		defer stopFetchEvents(fetchDelayedEventsWorker)
180
	}
181

182
	// Start moira escalations fetcher
183
	fetchEscalationsWorker := &escalations.FetchEscalationsWorker{
184
		Database: database,
185
		Notifier: sender,
186
	}
187
	fetchEscalationsWorker.Start()
188
	defer stopEscalationsFetcher(fetchEscalationsWorker)
189

190
	logger.InfoF("Moira Notifier Started. Version: %s", MoiraVersion)
191
	ch := make(chan os.Signal, 1)
192
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
193
	logger.Info(fmt.Sprint(<-ch))
194
	logger.Info("Moira Notifier shutting down.")
195
}
196

197
func stopFetchEvents(worker *events.FetchEventsWorker) {
198
	if err := worker.Stop(); err != nil {
199
		logger.ErrorF("Failed to stop events fetcher: %v", err)
200
	}
201
}
202

203
func stopNotificationsFetcher(worker *notifications.FetchNotificationsWorker) {
204
	if err := worker.Stop(); err != nil {
205
		logger.ErrorF("Failed to stop notifications fetcher: %v", err)
206
	}
207
}
208

209
func stopSelfStateChecker(checker *selfstate.SelfCheckWorker) {
210
	if err := checker.Stop(); err != nil {
211
		logger.ErrorF("Failed to stop self check worker: %v", err)
212
	}
213
}
214

215
func stopEscalationsFetcher(worker *escalations.FetchEscalationsWorker) {
216
	if err := worker.Stop(); err != nil {
217
		logger.ErrorF("Failed to stop escalations fetcher: %v", err)
218
	}
219
}
220