tetragon

1
// SPDX-License-Identifier: Apache-2.0
2
// Copyright Authors of Tetragon
3

4
// Tetragon bugtool code
5

6
package bugtool
7

8
import (
9
	"archive/tar"
10
	"bytes"
11
	"compress/gzip"
12
	"context"
13
	"encoding/json"
14
	"errors"
15
	"fmt"
16
	"io"
17
	"net"
18
	"net/http"
19
	"os"
20
	"os/exec"
21
	"path"
22
	"path/filepath"
23
	"strings"
24
	"time"
25

26
	"github.com/cilium/tetragon/api/v1/tetragon"
27
	"github.com/cilium/tetragon/pkg/defaults"
28
	"github.com/cilium/tetragon/pkg/logger"
29
	"github.com/cilium/tetragon/pkg/policyfilter"
30
	gopssignal "github.com/google/gops/signal"
31
	"go.uber.org/multierr"
32
	"google.golang.org/grpc"
33
	"google.golang.org/grpc/credentials/insecure"
34

35
	"github.com/sirupsen/logrus"
36
	"github.com/vishvananda/netlink"
37
)
38

39
// InitInfo contains information about how Tetragon was initialized.
40
type InitInfo struct {
41
	ExportFname string `json:"export_fname"`
42
	LibDir      string `json:"lib_dir"`
43
	BtfFname    string `json:"btf_fname"`
44
	ServerAddr  string `json:"server_address"`
45
	MetricsAddr string `json:"metrics_address"`
46
	GopsAddr    string `json:"gops_address"`
47
	MapDir      string `json:"map_dir"`
48
	BpfToolPath string `json:"bpftool_path"`
49
	GopsPath    string `json:"gops_path"`
50
}
51

52
// LoadInitInfo returns the InitInfo by reading the info file from its default location
53
func LoadInitInfo() (*InitInfo, error) {
54
	return doLoadInitInfo(defaults.InitInfoFile)
55
}
56

57
// SaveInitInfo saves InitInfo to the info file
58
func SaveInitInfo(info *InitInfo) error {
59
	return doSaveInitInfo(defaults.InitInfoFile, info)
60
}
61

62
func doLoadInitInfo(fname string) (*InitInfo, error) {
63
	f, err := os.Open(fname)
64
	if err != nil {
65
		logger.GetLogger().WithField("infoFile", fname).Warn("failed to open file")
66
		return nil, err
67
	}
68
	defer f.Close()
69

70
	var info InitInfo
71
	if err := json.NewDecoder(f).Decode(&info); err != nil {
72
		logger.GetLogger().WithField("infoFile", fname).Warn("failed to read information from file")
73
		return nil, err
74
	}
75

76
	return &info, nil
77
}
78

79
func doSaveInitInfo(fname string, info *InitInfo) error {
80
	// Complete InitInfo here
81
	bpftool, err := exec.LookPath("bpftool")
82
	if err != nil {
83
		logger.GetLogger().Warn("failed to locate bpftool binary, on bugtool debugging ensure you have bpftool installed")
84
	} else {
85
		info.BpfToolPath = bpftool
86
		logger.GetLogger().WithField("bpftool", info.BpfToolPath).Info("Successfully detected bpftool path")
87
	}
88

89
	gops, err := exec.LookPath("gops")
90
	if err != nil {
91
		logger.GetLogger().Warn("failed to locate gops binary, on bugtool debugging ensure you have gops installed")
92
	} else {
93
		info.GopsPath = gops
94
		logger.GetLogger().WithField("gops", info.GopsPath).Info("Successfully detected gops path")
95
	}
96

97
	// Create DefaultRunDir if it does not already exist
98
	if err := os.MkdirAll(defaults.DefaultRunDir, 0755); err != nil {
99
		logger.GetLogger().WithField("infoFile", fname).Warn("failed to directory exists")
100
		return err
101
	}
102
	f, err := os.OpenFile(fname, os.O_WRONLY|os.O_CREATE, 0744)
103
	if err != nil {
104
		logger.GetLogger().WithField("infoFile", fname).Warn("failed to create file")
105
		return err
106
	}
107
	defer f.Close()
108

109
	if err := f.Truncate(0); err != nil {
110
		logger.GetLogger().WithField("infoFile", fname).Warn("failed to truncate file")
111
		return err
112
	}
113

114
	if err := json.NewEncoder(f).Encode(info); err != nil {
115
		logger.GetLogger().WithField("infoFile", fname).Warn("failed to write information to file")
116
		return err
117
	}
118

119
	return nil
120
}
121

122
type bugtoolInfo struct {
123
	info      *InitInfo
124
	prefixDir string
125
	multiLog  MultiLog
126
}
127

128
func doTarAddBuff(tarWriter *tar.Writer, fname string, buff *bytes.Buffer) error {
129
	logHdr := tar.Header{
130
		Typeflag: tar.TypeReg,
131
		Name:     fname,
132
		Size:     int64(buff.Len()),
133
		Mode:     0644,
134
	}
135

136
	if err := tarWriter.WriteHeader(&logHdr); err != nil {
137
		logger.GetLogger().Error("failed to write log buffer tar header")
138
	}
139

140
	_, err := io.Copy(tarWriter, buff)
141
	if err != nil {
142
		logger.GetLogger().Error("failed to copy log buffer")
143
	}
144
	return err
145
}
146

147
func (s *bugtoolInfo) tarAddBuff(tarWriter *tar.Writer, fname string, buff *bytes.Buffer) error {
148
	name := filepath.Join(s.prefixDir, fname)
149
	return doTarAddBuff(tarWriter, name, buff)
150
}
151

152
func (s *bugtoolInfo) tarAddJson(tarWriter *tar.Writer, fname string, obj interface{}) error {
153
	b, err := json.Marshal(obj)
154
	if err != nil {
155
		return err
156
	}
157
	return s.tarAddBuff(tarWriter, fname, bytes.NewBuffer(b))
158
}
159

160
func (s *bugtoolInfo) tarAddFile(tarWriter *tar.Writer, fnameSrc string, fnameDst string) error {
161
	fileSrc, err := os.Open(fnameSrc)
162
	if err != nil {
163
		s.multiLog.WithField("path", fnameSrc).Warn("failed to open file")
164
		return err
165
	}
166
	defer fileSrc.Close()
167

168
	fileSrcInfo, err := fileSrc.Stat()
169
	if err != nil {
170
		s.multiLog.WithField("path", fnameSrc).Warn("failed to stat file")
171
		return err
172
	}
173

174
	hdr, err := tar.FileInfoHeader(fileSrcInfo, "" /* unused link target */)
175
	if err != nil {
176
		s.multiLog.Warn("error creating tar header")
177
		return err
178
	}
179
	hdr.Name = filepath.Join(s.prefixDir, fnameDst)
180

181
	if err := tarWriter.WriteHeader(hdr); err != nil {
182
		s.multiLog.Warn("failed to write tar header")
183
		return err
184
	}
185

186
	_, err = io.Copy(tarWriter, fileSrc)
187
	if err != nil {
188
		s.multiLog.WithField("fnameSrc", fnameSrc).Warn("error copying data from source file")
189
		return err
190
	}
191

192
	return nil
193
}
194

195
// Bugtool gathers information and writes it as a tar archive in the given filename
196
func Bugtool(outFname string, bpftool string, gops string) error {
197
	info, err := LoadInitInfo()
198
	if err != nil {
199
		return err
200
	}
201

202
	if bpftool != "" {
203
		info.BpfToolPath = bpftool
204
	}
205

206
	if gops != "" {
207
		info.GopsPath = gops
208
	}
209

210
	return doBugtool(info, outFname)
211
}
212

213
func doBugtool(info *InitInfo, outFname string) error {
214
	// we log into two logs, one is the standard one and another one is a
215
	// buffer that we are going to include as a file into the bugtool archive.
216
	bugtoolLogger := logrus.New()
217
	logBuff := new(bytes.Buffer)
218
	bugtoolLogger.Out = logBuff
219
	logrus.SetLevel(logrus.InfoLevel)
220
	multiLog := MultiLog{
221
		Logs: []logrus.FieldLogger{
222
			logger.GetLogger(),
223
			bugtoolLogger,
224
		},
225
	}
226
	prefixDir := fmt.Sprintf("tetragon-bugtool-%s", time.Now().Format("20060102150405"))
227

228
	outFile, err := os.Create(outFname)
229
	if err != nil {
230
		multiLog.WithError(err).WithField("tarFile", outFname).Warn("failed to create bugtool tarfile")
231
		return err
232
	}
233
	defer outFile.Close()
234

235
	si := bugtoolInfo{
236
		info:      info,
237
		prefixDir: prefixDir,
238
		multiLog:  multiLog,
239
	}
240

241
	gzWriter := gzip.NewWriter(outFile)
242
	defer gzWriter.Close()
243

244
	tarWriter := tar.NewWriter(gzWriter)
245
	defer func() {
246
		defer tarWriter.Close()
247
		si.tarAddBuff(tarWriter, "tetragon-bugtool.log", logBuff)
248
	}()
249

250
	si.addInitInfo(tarWriter)
251
	si.addLibFiles(tarWriter)
252
	si.addBtfFile(tarWriter)
253
	si.addTetragonLog(tarWriter)
254
	si.addMetrics(tarWriter)
255
	si.execCmd(tarWriter, "dmesg.out", "dmesg")
256
	si.addTcInfo(tarWriter)
257
	si.addBpftoolInfo(tarWriter)
258
	si.addGopsInfo(tarWriter)
259
	si.dumpPolicyFilterMap(tarWriter)
260
	si.addGrpcInfo(tarWriter)
261
	return nil
262
}
263

264
func (s *bugtoolInfo) addInitInfo(tarWriter *tar.Writer) error {
265
	s.multiLog.Info("saving init info")
266
	buff := new(bytes.Buffer)
267
	if err := json.NewEncoder(buff).Encode(s.info); err != nil {
268
		s.multiLog.Warn("failed to serialze init info")
269
		return err
270
	}
271
	return s.tarAddBuff(tarWriter, "tetragon-info.json", buff)
272
}
273

274
// addLibFiles adds all files under the hubble lib directory to the archive.
275
//
276
// Currently, this includes the bpf files and potentially the btf file if it is stored there.  If
277
// there are files that we do not want to add, we can filter them out, but for now we can just grab
278
// everything.
279
func (s *bugtoolInfo) addLibFiles(tarWriter *tar.Writer) error {
280
	s.multiLog.WithField("libDir", s.info.LibDir).Info("retrieving lib directory")
281
	return filepath.Walk(
282
		s.info.LibDir,
283
		// NB: if the walk function returns an error, the walk terminates.
284
		// We want to gather as much information as possible, so we
285
		// never return an error.
286
		func(path string, info os.FileInfo, err error) error {
287
			if err != nil {
288
				s.multiLog.WithField("path", path).Warn("error walking path.")
289
				return nil
290
			}
291

292
			if info.IsDir() && info.Name() == "metadata" {
293
				s.multiLog.WithField("path", path).Info("skipping metadata directory")
294
				return filepath.SkipDir
295
			}
296

297
			// We ignore non-regular files.
298
			// Note that this also includes symbolic links. We could be smarter about
299
			// symlinks if they point within the directory we are archiving, but since
300
			// we do not use them, there is currently no reason for the complexity.
301
			mode := info.Mode()
302
			if !(mode.IsRegular() || mode.IsDir()) {
303
				s.multiLog.WithField("path", path).Warn("not a regular file, ignoring")
304
				return nil
305
			}
306

307
			if !strings.HasSuffix(info.Name(), ".o") {
308
				s.multiLog.WithField("path", path).Warn("not an object file, ignoring")
309
				return nil
310
			}
311

312
			hdr, err := tar.FileInfoHeader(info, "" /* unused link target */)
313
			if err != nil {
314
				s.multiLog.WithField("path", path).Warn("error creating tar header")
315
				return nil
316
			}
317
			// fix filename
318
			hdr.Name = filepath.Join(s.prefixDir, "lib", strings.TrimPrefix(path, s.info.LibDir))
319

320
			if err := tarWriter.WriteHeader(hdr); err != nil {
321
				s.multiLog.WithField("path", path).Warn("failed to write tar header")
322
				return nil
323
			}
324

325
			if info.IsDir() {
326
				return nil
327
			}
328

329
			// open and copy file to the tar archive
330
			file, err := os.Open(path)
331
			if err != nil {
332
				s.multiLog.WithField("path", path).Warn("error opening file")
333
				return nil
334
			}
335
			defer file.Close()
336
			_, err = io.Copy(tarWriter, file)
337
			if err != nil {
338
				s.multiLog.WithField("path", path).Warn("error copying data from file")
339
				return nil
340
			}
341
			return nil
342
		})
343
}
344

345
// addBtfFile adds the btf file to the archive.
346
func (s *bugtoolInfo) addBtfFile(tarWriter *tar.Writer) error {
347
	btfFname, err := filepath.EvalSymlinks(s.info.BtfFname)
348
	if err != nil && s.info.BtfFname != "" {
349
		s.multiLog.WithField("btfFname", s.info.BtfFname).Warnf("error resolving btf file: %s", err)
350
		return err
351
	}
352

353
	if s.info.BtfFname == "" {
354
		s.multiLog.Warnf("no btf filename in tetragon config, attempting to fall back to /sys/kernel/btf/vmlinux")
355
		btfFname = "/sys/kernel/btf/vmlinux"
356
	}
357

358
	if rel, err := filepath.Rel(s.info.LibDir, btfFname); err == nil && !strings.HasPrefix(rel, "..") {
359
		s.multiLog.WithField("btfFname", btfFname).Infof("btf file already in lib dir: %s", rel)
360
		return nil
361
	}
362

363
	err = s.tarAddFile(tarWriter, btfFname, "btf")
364
	if err == nil {
365
		s.multiLog.WithField("btfFname", btfFname).Info("btf file added")
366
	}
367
	return err
368
}
369

370
// addTetragonLog adds the tetragon log file to the archive
371
func (s *bugtoolInfo) addTetragonLog(tarWriter *tar.Writer) error {
372
	if s.info.ExportFname == "" {
373
		s.multiLog.Info("no export file specified")
374
		return nil
375
	}
376

377
	err := s.tarAddFile(tarWriter, s.info.ExportFname, "tetragon.log")
378
	if err == nil {
379
		s.multiLog.WithField("exportFname", s.info.ExportFname).Info("tetragon log file added")
380
	}
381
	return err
382
}
383

384
// addMetrics adds the output of metrics in the tar file
385
func (s *bugtoolInfo) addMetrics(tarWriter *tar.Writer) error {
386
	// nothing to do if metrics server is not running
387
	if s.info.MetricsAddr == "" {
388
		return nil
389
	}
390

391
	// determine the port that the metrics server listens to
392
	slice := strings.Split(s.info.MetricsAddr, ":")
393
	if len(slice) < 2 {
394
		s.multiLog.WithField("metricsAddr", s.info.MetricsAddr).Warn("could not determine metrics port")
395
		return errors.New("failed to determine metrics port")
396
	}
397
	port := slice[len(slice)-1]
398

399
	// contact metrics server
400
	metricsAddr := fmt.Sprintf("http://localhost:%s/metrics", port)
401
	s.multiLog.WithField("metricsAddr", metricsAddr).Info("contacting metrics server")
402
	resp, err := http.Get(metricsAddr)
403
	if err != nil {
404
		s.multiLog.WithField("metricsAddr", metricsAddr).WithField("err", err).Warn("failed to contact metrics server")
405
		return err
406
	}
407
	defer resp.Body.Close()
408

409
	buff := new(bytes.Buffer)
410
	if _, err = buff.ReadFrom(resp.Body); err != nil {
411
		s.multiLog.Warn("error in reading metrics server response: %s", err)
412
	}
413
	return s.tarAddBuff(tarWriter, "metrics", buff)
414
}
415

416
// execCmd executes a command and saves its output (both stdout and stderr) to a file in the tar archive
417
func (s *bugtoolInfo) execCmd(tarWriter *tar.Writer, dstFname string, cmdName string, cmdArgs ...string) error {
418
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
419
	defer cancel()
420
	cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
421

422
	stdin, err := cmd.StdinPipe()
423
	if err != nil {
424
		s.multiLog.Warnf("StdinPipe() failed: %s", err)
425
		return err
426
	}
427
	stdin.Close()
428

429
	stdout, err := cmd.StdoutPipe()
430
	if err != nil {
431
		s.multiLog.Warnf("StdoutPipe() failed: %v", err)
432
		return err
433
	}
434

435
	stderr, err := cmd.StderrPipe()
436
	if err != nil {
437
		s.multiLog.Warnf("StderrPipe() failed: %v", err)
438
		return err
439
	}
440

441
	err = cmd.Start()
442
	if err != nil {
443
		s.multiLog.WithField("cmd", cmd).WithError(err).Warnf("failed to execute command")
444
		return err
445
	}
446
	// NB: copying everything to a buffer makes things easier because we can
447
	// compute the size of the file we want to write to the tar archive.
448
	// If, however, we use this with programs (not currently the case) that
449
	// have outputs too large for memory, this would be problematic because
450
	// it will lead to swapping or OOM.
451
	outbuff := new(bytes.Buffer)
452
	if _, err = outbuff.ReadFrom(stdout); err != nil {
453
		s.multiLog.WithField("cmd", cmd).WithError(err).Warnf("error reading stdout")
454
	}
455

456
	errbuff := new(bytes.Buffer)
457
	if _, err = errbuff.ReadFrom(stderr); err != nil {
458
		s.multiLog.WithField("cmd", cmd).WithError(err).Warnf("error reading stderr")
459
	}
460

461
	errStr := "0"
462
	err = cmd.Wait()
463
	if err != nil {
464
		errStr = err.Error()
465
	}
466
	s.multiLog.WithField("cmd", cmd).WithField("ret", errStr).WithField("dstFname", dstFname).Info("executed command")
467

468
	ret := s.tarAddBuff(tarWriter, dstFname, outbuff)
469
	if errbuff.Len() > 0 {
470
		errstderr := s.tarAddBuff(tarWriter, dstFname+".err", errbuff)
471
		ret = multierr.Append(ret, errstderr)
472
	}
473
	return ret
474
}
475

476
// addTcInfo adds information about tc filters on the devices
477
func (s *bugtoolInfo) addTcInfo(tarWriter *tar.Writer) error {
478
	links, err := netlink.LinkList()
479
	if err != nil {
480
		s.multiLog.WithError(err).Warn("listing devices failed")
481
		return err
482
	}
483

484
	// NB: We could save the interfaces that tetragon installed programs and
485
	// query only those by saving the interfaces to the info file. Instead,
486
	// we perform the command for all links in the system. This is simpler
487
	// and also provides additional information that may be useful.
488
	for _, link := range links {
489
		linkName := link.Attrs().Name
490
		s.execCmd(tarWriter, fmt.Sprintf("tc-info.%s.ingress", linkName), "tc", "filter", "show", "dev", linkName, "ingress")
491
		s.execCmd(tarWriter, fmt.Sprintf("tc-info.%s.egress", linkName), "tc", "filter", "show", "dev", linkName, "egress")
492
	}
493

494
	return err
495
}
496

497
// addBpftoolInfo adds information about loaded eBPF maps and programs
498
func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) {
499
	if s.info.BpfToolPath == "" {
500
		s.multiLog.Warn("Failed to locate bpftool, please install it and specify its path")
501
		return
502
	}
503

504
	_, err := os.Stat(s.info.BpfToolPath)
505
	if err != nil {
506
		s.multiLog.WithError(err).Warn("Failed to locate bpftool. Please install it or specify its path, see 'bugtool --help'")
507
		return
508
	}
509
	s.execCmd(tarWriter, "bpftool-maps.json", s.info.BpfToolPath, "map", "show", "-j")
510
	s.execCmd(tarWriter, "bpftool-progs.json", s.info.BpfToolPath, "prog", "show", "-j")
511
	s.execCmd(tarWriter, "bpftool-cgroups.json", s.info.BpfToolPath, "cgroup", "tree", "-j")
512
}
513

514
func (s *bugtoolInfo) getPProf(tarWriter *tar.Writer, file string) error {
515
	if s.info.GopsAddr == "" {
516
		s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops")
517
		return nil
518
	}
519

520
	s.multiLog.WithField("gops-address", s.info.GopsAddr).Info("Contacting gops server for pprof dump")
521

522
	conn, err := net.Dial("tcp", s.info.GopsAddr)
523
	if err != nil {
524
		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to contact gops server")
525
		return err
526
	}
527

528
	buf := []byte{gopssignal.HeapProfile}
529
	if _, err := conn.Write(buf); err != nil {
530
		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to send gops pprof-heap command")
531
		return err
532
	}
533

534
	buff := new(bytes.Buffer)
535
	if _, err = buff.ReadFrom(conn); err != nil {
536
		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed reading gops pprof-heap response")
537
	}
538
	return s.tarAddBuff(tarWriter, file, buff)
539
}
540

541
func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) {
542
	if s.info.GopsAddr == "" {
543
		s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops")
544
		return
545
	}
546

547
	if s.info.GopsPath == "" {
548
		s.multiLog.WithField("gops-address", s.info.GopsAddr).Warn("Failed to locate gops. Please install it or specify its path, see 'bugtool --help'")
549
		return
550
	}
551

552
	_, err := os.Stat(s.info.GopsPath)
553
	if err != nil {
554
		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it")
555
		return
556
	}
557

558
	s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Dumping gops information")
559

560
	s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr)
561
	s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr)
562
	s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr)
563
	err = s.getPProf(tarWriter, "gops.pprof-heap")
564
	if err != nil {
565
		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).WithError(err).Warn("Failed to dump gops pprof-heap")
566
	} else {
567
		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Successfully dumped gops pprof-heap")
568
	}
569
}
570

571
func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error {
572
	fname := path.Join(s.info.MapDir, policyfilter.MapName)
573
	m, err := policyfilter.OpenMap(fname)
574
	if err != nil {
575
		s.multiLog.WithError(err).Warnf("failed to open policyfilter map")
576
		return err
577
	}
578

579
	obj, err := m.Dump()
580
	if err != nil {
581
		s.multiLog.WithError(err).Warnf("failed to dump policyfilter map")
582
		return err
583
	}
584
	return s.tarAddJson(tarWriter, policyfilter.MapName+".json", obj)
585
}
586

587
func (s *bugtoolInfo) addGrpcInfo(tarWriter *tar.Writer) {
588
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
589
	defer cancel()
590
	conn, err := grpc.DialContext(
591
		ctx,
592
		s.info.ServerAddr,
593
		grpc.WithTransportCredentials(insecure.NewCredentials()),
594
		grpc.WithBlock(),
595
	)
596
	if err != nil {
597
		s.multiLog.Warnf("failed to connect to %s: %v", s.info.ServerAddr, err)
598
		return
599
	}
600
	defer conn.Close()
601
	client := tetragon.NewFineGuidanceSensorsClient(conn)
602

603
	res, err := client.ListTracingPolicies(ctx, &tetragon.ListTracingPoliciesRequest{})
604
	if err != nil || res == nil {
605
		s.multiLog.Warnf("failed to list tracing policies: %v", err)
606
		return
607
	}
608

609
	fname := "tracing-policies.json"
610
	err = s.tarAddJson(tarWriter, fname, res)
611
	if err != nil {
612
		s.multiLog.Warnf("failed to dump tracing policies: %v", err)
613
		return
614
	}
615

616
	s.multiLog.Infof("dumped tracing policies in %s", fname)
617
}
618