pupirka

1
package main
2

3
import (
4
	"encoding/json"
5
	"errors"
6
	"fmt"
7
	"github.com/gammazero/workerpool"
8
	logrus "github.com/sirupsen/logrus"
9
	"io/ioutil"
10
	"os"
11
	"regexp"
12
	"strconv"
13
	"strings"
14
	"time"
15
)
16

17
type Logger logrus.Logger
18

19
func ScanDevice() {
20
	dir := ConfigV.GetString("path.devices")
21

22
	files, err := ioutil.ReadDir(dir)
23
	if err != nil {
24
		es := fmt.Sprintf(err.Error())
25
		LogConsole.Error(es)
26
		LogConsole.Error("Program exit")
27
		os.Exit(1)
28
	}
29

30
	for _, f := range files {
31
		fr := regexp.MustCompile(`\.json$`)
32
		if frr := string(fr.Find([]byte(f.Name()))); frr == "" {
33
			es := fmt.Sprintf("File %s skipped, not valid file extension", f.Name())
34
			LogConsole.Error(es)
35
			continue
36
		}
37
		DeviceFiles = append(DeviceFiles, f.Name())
38
	}
39
}
40

41
func ReadDevice(Dev *DeviceList) {
42
	for _, f := range DeviceFiles {
43
		filepath := fmt.Sprintf("%s/%s", ConfigV.GetString("path.devices"), f)
44
		jsonFile, err := os.Open(filepath)
45
		if err != nil {
46
			es := fmt.Sprintf("Error file Open %s, Error:%s, Skip file", f, err.Error())
47
			LogConsole.Error(es)
48
			continue
49
		}
50
		defer jsonFile.Close()
51

52
		byteValueFromFile, err := ioutil.ReadAll(jsonFile)
53
		if err != nil {
54

55
			es := fmt.Sprintf("Error file Read %s, Error:%s, Skip file", f, err.Error())
56
			LogConsole.Error(es)
57
			jsonFile.Close()
58
			continue
59
		}
60
		var d Device
61
		err = json.Unmarshal(byteValueFromFile, &d)
62
		if err != nil {
63
			es := fmt.Sprintf("Error Read file json  %s, Error:%s, Skip file", f, err.Error())
64
			LogConsole.Error(es)
65
			jsonFile.Close()
66
			continue
67
		}
68
		d.Name = f[:len(f)-5]
69
		d.LogConfig()
70
		SetDefaultParameter(&d)
71

72
		MDeviceList[d.Name] = d
73
		Dev.Devices = append(Dev.Devices, d)
74
	}
75
}
76

77
func RotateDevice(device *DeviceList) {
78
	LogConsole.Info("Rotate device list...")
79

80
	for i, d := range device.Devices {
81
		d.LogDebug("RotateDevice: check directory backup", d.Name, d.Dirbackup)
82
		if _, err := os.Stat(d.Dirbackup); os.IsNotExist(err) {
83
			_ = os.Mkdir(d.Dirbackup, os.ModePerm)
84
			LogConsole.Info(fmt.Sprintf("Create Folder %s for backup", d.Dirbackup))
85
			continue
86
		}
87
		d.LogDebug("RotateDevice: read folder backup", d.Name, d.Dirbackup)
88
		files, err := ioutil.ReadDir(d.Dirbackup)
89
		if err != nil {
90
			es := fmt.Sprintf("Error read folder backup %s, Error:%s", d.Dirbackup, err.Error())
91
			LogConsole.Error(es)
92
			device.Devices[i].StatusJob = "skip"
93
			continue
94
		}
95
		d.LogDebug("RotateDevice: Find old backup", d.Name, d.Dirbackup)
96
		for _, f := range files {
97
			d.LogDebug(fmt.Sprintf("RotateDevice: Check File %s, time %s", f.Name(), f.ModTime().String()))
98
			now := time.Now()
99
			fdifftimesecond := now.Sub(f.ModTime()).Seconds()
100
			diffday := fdifftimesecond / 60 / 60 / 24
101
			if int(diffday) > d.Rotate {
102
				d.LogDebug(fmt.Sprintf("RotateDevice: File %s old backup for device %s", f.Name(), d.Name))
103
				if len(files) > 5 {
104
					d.LogDebug("RotateDevice: File Count > 5 in backup", d.Name)
105
					d.LogDebug("RotateDevice: Remove old files", f.Name())
106
					filename := fmt.Sprintf("%s/%s", d.Dirbackup, f.Name())
107
					err := os.Remove(filename)
108
					if err != nil {
109
						es := fmt.Sprintf("Error Remove file %s, Error:%s", f.Name(), err.Error())
110
						LogConsole.Error(es)
111
						continue
112
					}
113
					if err := gitClient.RemoveFile(filename); err != nil {
114
						d.LogWarn(err)
115
						continue
116
					}
117
				}
118

119
			}
120
			if int(fdifftimesecond) < d.Every {
121
				d.LogDebug(fmt.Sprintf("RotateDevice: control check file time (%f), need time (%d)", fdifftimesecond, d.Every))
122
				d.LogDebug("RotateDevice: Device no need backup", d.Name)
123
				device.Devices[i].StatusJob = "skip"
124
				break
125
			}
126

127
		}
128

129
	}
130
	/*
131
		newDev := DeviceList{}
132
		for _, d := range device.Devices {
133
			if d.Name == "" {
134
				continue
135
			}
136
			newDev.Devices = append(newDev.Devices, d)
137
		}
138
		device.Devices = newDev.Devices
139
	*/
140
}
141

142
func RunBackups(Dev *DeviceList) {
143
	LogConsole.Info("Backup Start ---->")
144
	//todo need create dynamic group, curent group 10 and wait all execute after next 10.
145
	//todo need if one exit group, send one next to group
146
	wp := workerpool.New(ConfigV.GetInt("process.max"))
147
	for _, d := range Dev.Devices {
148
		d := d
149
		if d.StatusJob == "skip" {
150
			d.Hook()
151
			continue
152
		}
153
		d.LogDebug("RunBackups: wait groups", d.Name)
154
		wp.Submit(func() {
155
			d.LogDebug("RunBackups: enter groups", d.Name)
156
			backup(&d)
157
			d.Hook()
158
		})
159
		d.LogDebug("RunBackups: Exit groups", d.Name)
160
	}
161
	wp.StopWait()
162
	LogConsole.Info("Backup Finish <----")
163
}
164

165
func backup(device *Device) {
166

167
	LogConsole.Warn(fmt.Sprintf("Starting backup %s ...", device.Name))
168
	device.LogDebug("Starting backup ", device.Name)
169
	var bytefromsshclient []byte
170

171
	if device.Parent != "" {
172
		device.LogDebug("backup: parent exist ", device.Name)
173
		parent, child, err := SshNeedForward(device)
174
		if err != nil {
175
			ers := fmt.Sprintf("Fatal error Device:%s get parent %s error: %s", device.Name, device.Parent, err.Error())
176
			LogConsole.Error(ers)
177
			device.LogError(ers)
178
			device.StatusJob = "error"
179
			return
180
		}
181

182
		newd := SshForwardNewDevice(parent, child)
183

184
		bytefromsshclient, err = SshClientRun(&newd)
185
		if err != nil {
186
			ers := fmt.Sprintf("Fatal error Forward Device:%s: %s", device.Name, err.Error())
187
			LogConsole.Error(ers)
188
			device.LogError(ers)
189
			device.StatusJob = "error"
190
			return
191
		}
192
	} else {
193
		device.LogDebug("backup: no parent ", device.Name)
194
		bytefromssh, err := SshClientRun(device)
195
		if err != nil {
196
			ers := fmt.Sprintf("Fatal error Device:%s: %s", device.Name, err.Error())
197
			LogConsole.Error(ers)
198
			device.LogError(ers)
199
			device.StatusJob = "error"
200
			return
201
		}
202
		bytefromsshclient = bytefromssh
203
	}
204

205
	if bytefromsshclient == nil {
206
		ers := fmt.Sprintf("Fatal error Device:%s not bytes for backup", device.Name)
207
		LogConsole.Error(ers)
208
		device.LogError(ers)
209
		device.StatusJob = "error"
210
		return
211
	}
212
	err := SaveBackupFile(device, bytefromsshclient)
213
	if err != nil {
214
		ers := fmt.Sprintf("Bad saved config device %s Error: %s", device.Name, err.Error())
215
		LogConsole.Error(ers)
216
		device.LogError(ers)
217
		device.StatusJob = "error"
218
		return
219
	}
220
	device.StatusJob = "backup"
221
	device.LogInfo("Backup complete")
222

223
}
224

225
func SaveBackupFile(device *Device, b []byte) error {
226

227
	device.LogDebug(fmt.Sprintf("SaveBackupFile: saved backups... %s", device.Name))
228
	backupfile := fmt.Sprintf("%s/%s", device.Dirbackup, device.BackupFileName)
229
	FileBackupExist := FileExistBool(backupfile)
230
	result := b
231
	if device.Clearstring != "" {
232
		device.LogDebug(fmt.Sprintf("SaveBackupFile: Need clear string in config... %s", device.Name))
233
		result = RemoveStringFromBakcup(device, b)
234
	}
235
	if FileBackupExist == true {
236
		com, err := FileCompareByteBool(backupfile, result)
237
		if err != nil {
238
			return err
239
		}
240
		if com == true {
241
			return nil
242
		}
243
	}
244

245
	var fn *os.File
246

247
	var err error
248
	if FileBackupExist == false {
249
		device.LogDebug(fmt.Sprintf("SaveBackupFile: Create file... %s", backupfile))
250
		fn, err = os.Create(backupfile)
251
		if err != nil {
252
			return errors.New(fmt.Sprintf("SaveBackupFile: Create file Error:%s", err.Error()))
253
		}
254
	} else {
255
		fn, err = os.OpenFile(backupfile, os.O_RDWR|os.O_CREATE, 0755)
256
		if err != nil {
257
			return errors.New(fmt.Sprintf("SaveBackupFile: Open file Error:%s", err.Error()))
258
		}
259
	}
260

261
	device.LogDebug(fmt.Sprintf("SaveBackupFile: Write to file... %s", backupfile))
262
	_, err = fn.Write(result)
263
	if err != nil {
264
		return errors.New(fmt.Sprintf("SaveBackupFile: Write to file Error:%s", err.Error()))
265
	}
266
	device.LogDebug(fmt.Sprintf("SaveBackupFile: Close file... %s", backupfile))
267
	_ = fn.Close()
268
	if FileBackupExist {
269
		//CommitName := fmt.Sprintf("Change backup in device %s", device.Name)
270
		if err := gitClient.SetCommit(backupfile); err != nil {
271
			device.LogWarn(err)
272
			LogConsole.Error(err)
273
			return err
274
		}
275

276
	} else {
277
		if err := gitClient.AddFile(backupfile); err != nil {
278
			device.LogWarn(err)
279
			LogConsole.Error(err)
280
			return err
281
		}
282
	}
283

284
	return nil
285
}
286

287
func RemoveStringFromBakcup(device *Device, b []byte) []byte {
288
	device.LogDebug(fmt.Sprintf("RemoveStringFromBackup: ... %s", device.Name))
289
	device.LogDebug(fmt.Sprintf("RemoveStringFromBackup: ... %s", device.Name))
290
	regstr := fmt.Sprintf(`(?m:^%s.*$)`, device.Clearstring)
291
	device.LogDebug(fmt.Sprintf("RemoveStringFromBackup: regexp %s", regstr))
292
	re := regexp.MustCompile(regstr)
293
	device.LogDebug(fmt.Sprintf("RemoveStringFromBackup: replace string %s", device.Name))
294
	config := re.ReplaceAllString(string(b), "")
295
	config = strings.Trim(config, "\r\n")
296
	device.LogDebug(fmt.Sprintf("RemoveStringFromBackup: Remove empty string %s", device.Name))
297
	return []byte(config)
298
}
299

300
func SetDefaultParameter(d *Device) {
301

302
	if d.Timeout == 0 {
303

304
		d.Timeout = ConfigV.GetInt("devicedefault.timeout")
305
		d.LogDebug("SetDefaultParameter: Set default Timeout ", d.Name, d.Timeout)
306
	}
307
	if d.Every == 0 {
308

309
		d.Every = ConfigV.GetInt("devicedefault.every")
310

311
		d.LogDebug("SetDefaultParameter: Set default Every ", d.Name, d.Every)
312
	}
313
	if d.Rotate == 0 {
314

315
		d.Rotate = ConfigV.GetInt("devicedefault.rotate")
316
		d.LogDebug("SetDefaultParameter: Set default Rotate ", d.Name, d.Rotate)
317
	}
318
	if d.Command == "" {
319

320
		d.Command = ConfigV.GetString("devicedefault.command")
321
		d.LogDebug("SetDefaultParameter: Set default Command ", d.Name, d.Command)
322
	}
323
	if d.PortSSH == 0 {
324
		d.LogDebug("SetDefaultParameter: Set default PortSSH", d.Name)
325
		p, err := strconv.Atoi(ConfigV.GetString("devicedefault.portshh"))
326
		if err != nil {
327

328
			d.PortSSH = 22
329
			d.LogDebug("SetDefaultParameter: Error parse ssh port field", d.PortSSH)
330
		}
331

332
		if err == nil || p < 65535 || p > 0 {
333
			d.PortSSH = uint16(p)
334
			d.LogDebug("SetDefaultParameter: Set default PortSSH ", d.Name, d.PortSSH)
335
		} else {
336

337
			d.PortSSH = 22
338
			d.LogDebug("SetDefaultParameter: Error port ssh not range 1-65535", d.Name, d.PortSSH)
339
		}
340
	}
341
	d.Authkey = false
342
	if d.Key != "" {
343
		d.LogDebug("SetDefaultParameter: Need use private key uniq ", d.Name, d.Key)
344
		d.Authkey = true
345
	}
346

347
	if d.Authkey == false && d.Password == "" && d.Key != "" {
348
		d.LogDebug("SetDefaultParameter: Need use private key uniq ", d.Name, d.Key)
349
		d.Authkey = true
350
	}
351
	if d.Authkey == false && d.Password == "" && d.Key == "" && ConfigV.GetString("devicedefault.key") != "" {
352

353
		d.Authkey = true
354
		d.Key = ConfigV.GetString("devicedefault.key")
355
		d.LogDebug("SetDefaultParameter: Need use private key default ", d.Name, d.Key)
356
	}
357

358
	if d.TimeFormat == "" {
359
		d.TimeFormat = ConfigV.GetString("devicedefault.timeformat")
360
		d.LogDebug("SetDefaultParameter: Set default time format ", d.Name, d.TimeFormat)
361
	}
362

363
	if d.Prefix == "" {
364
		d.Prefix = ConfigV.GetString("devicedefault.prefix")
365
		d.LogDebug("SetDefaultParameter: Set default Prefix", d.Name, d.Prefix)
366
	}
367
	if d.FileNameFormat == "" {
368
		d.FileNameFormat = ConfigV.GetString("devicedefault.filenameformat")
369
		d.LogDebug("SetDefaultParameter: Set default FileNameFormat", d.Name, d.FileNameFormat)
370
	}
371

372
	if d.Clearstring == "" {
373
		d.Clearstring = ConfigV.GetString("devicedefault.clearstring")
374
		d.LogDebug("SetDefaultParameter: Set default Clear string", d.Name, d.Clearstring)
375
	}
376
	d.Dirbackup = fmt.Sprintf("%s/%s", ConfigV.GetString("path.backup"), d.Name)
377
	ditetimestring := time.Now().Format(d.TimeFormat)
378
	nameinprefix := strings.ReplaceAll(d.FileNameFormat, "%p", d.Prefix)
379
	nameintime := strings.ReplaceAll(nameinprefix, "%t", ditetimestring)
380
	nameinname := strings.ReplaceAll(nameintime, "%n", d.Name)
381
	d.BackupFileName = nameinname
382
	d.LogDebug("SetDefaultParameter: Set default BackupFileName", d.Name, d.BackupFileName)
383
	if d.DeviceHooks.Error == "" {
384
		d.DeviceHooks.Error = ConfigV.GetString("devicedefault.hook.error")
385
		d.LogDebug(fmt.Sprintf("SetDefaultParameter: Set default devicedefault.hook.error Velue %s in Device %s", d.DeviceHooks.Error, d.Name))
386
	}
387
	if d.DeviceHooks.Backup == "" {
388
		d.DeviceHooks.Backup = ConfigV.GetString("devicedefault.hook.backup")
389
		d.LogDebug(fmt.Sprintf("SetDefaultParameter: Set default devicedefault.hook.backup Velue %s in Device %s", d.DeviceHooks.Backup, d.Name))
390
	}
391
	if d.DeviceHooks.Skip == "" {
392
		d.DeviceHooks.Skip = ConfigV.GetString("devicedefault.hook.skip")
393
		d.LogDebug(fmt.Sprintf("SetDefaultParameter: Set default devicedefault.hook.skip Velue %s in Device %s", d.DeviceHooks.Skip, d.Name))
394
	}
395
}
396