istio

1
// Copyright Istio Authors
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14

15
package bugreport
16

17
import (
18
	"context"
19
	"fmt"
20
	"os"
21
	"path"
22
	"path/filepath"
23
	"reflect"
24
	"runtime"
25
	"strings"
26
	"sync"
27
	"time"
28

29
	"github.com/kr/pretty"
30
	"github.com/spf13/cobra"
31

32
	label2 "istio.io/api/label"
33
	"istio.io/istio/istioctl/pkg/cli"
34
	"istio.io/istio/istioctl/pkg/util/ambient"
35
	"istio.io/istio/operator/pkg/util"
36
	"istio.io/istio/pkg/kube"
37
	"istio.io/istio/pkg/kube/inject"
38
	"istio.io/istio/pkg/log"
39
	"istio.io/istio/pkg/proxy"
40
	"istio.io/istio/pkg/util/sets"
41
	"istio.io/istio/pkg/version"
42
	"istio.io/istio/tools/bug-report/pkg/archive"
43
	cluster2 "istio.io/istio/tools/bug-report/pkg/cluster"
44
	"istio.io/istio/tools/bug-report/pkg/common"
45
	"istio.io/istio/tools/bug-report/pkg/config"
46
	"istio.io/istio/tools/bug-report/pkg/content"
47
	"istio.io/istio/tools/bug-report/pkg/filter"
48
	"istio.io/istio/tools/bug-report/pkg/kubeclient"
49
	"istio.io/istio/tools/bug-report/pkg/kubectlcmd"
50
	"istio.io/istio/tools/bug-report/pkg/processlog"
51
)
52

53
const (
54
	bugReportDefaultTimeout = 30 * time.Minute
55
)
56

57
var (
58
	bugReportDefaultIstioNamespace = "istio-system"
59
	bugReportDefaultInclude        = []string{""}
60
	bugReportDefaultExclude        = []string{strings.Join(sets.SortedList(inject.IgnoredNamespaces), ",")}
61
)
62

63
// Cmd returns a cobra command for bug-report.
64
func Cmd(ctx cli.Context, logOpts *log.Options) *cobra.Command {
65
	rootCmd := &cobra.Command{
66
		Use:          "bug-report",
67
		Short:        "Cluster information and log capture support tool.",
68
		SilenceUsage: true,
69
		Long: `bug-report selectively captures cluster information and logs into an archive to help diagnose problems.
70
Proxy logs can be filtered using:
71
  --include|--exclude ns1,ns2.../dep1,dep2.../pod1,pod2.../lbl1=val1,lbl2=val2.../ann1=val1,ann2=val2.../cntr1,cntr...
72
where ns=namespace, dep=deployment, lbl=label, ann=annotation, cntr=container
73

74
The filter spec is interpreted as 'must be in (ns1 OR ns2) AND (dep1 OR dep2) AND (cntr1 OR cntr2)...'
75
The log will be included only if the container matches at least one include filter and does not match any exclude filters.
76
All parts of the filter are optional and can be omitted e.g. ns1//pod1 filters only for namespace ns1 and pod1.
77
All names except label and annotation keys support '*' glob matching pattern.
78

79
e.g.
80
--include ns1,ns2 (only namespaces ns1 and ns2)
81
--include n*//p*/l=v* (pods with name beginning with 'p' in namespaces beginning with 'n' and having label 'l' with value beginning with 'v'.)`,
82
		RunE: func(cmd *cobra.Command, args []string) error {
83
			return runBugReportCommand(ctx, cmd, logOpts)
84
		},
85
	}
86
	rootCmd.AddCommand(version.CobraCommand())
87
	addFlags(rootCmd, gConfig)
88

89
	return rootCmd
90
}
91

92
var (
93
	// Logs, along with stats and importance metrics. Key is path (namespace/deployment/pod/cluster) which can be
94
	// parsed with ParsePath.
95
	logs       = make(map[string]string)
96
	stats      = make(map[string]*processlog.Stats)
97
	importance = make(map[string]int)
98
	// Aggregated errors for all fetch operations.
99
	gErrors util.Errors
100
	lock    = sync.RWMutex{}
101
)
102

103
func runBugReportCommand(ctx cli.Context, _ *cobra.Command, logOpts *log.Options) error {
104
	runner := kubectlcmd.NewRunner(gConfig.RequestConcurrency)
105
	runner.ReportRunningTasks()
106
	if err := configLogs(logOpts); err != nil {
107
		return err
108
	}
109
	config, err := parseConfig()
110
	if err != nil {
111
		return err
112
	}
113
	clusterCtxStr := ""
114
	if config.Context == "" {
115
		var err error
116
		clusterCtxStr, err = content.GetClusterContext(runner, config.KubeConfigPath)
117
		if err != nil {
118
			return err
119
		}
120
	} else {
121
		clusterCtxStr = config.Context
122
	}
123

124
	common.LogAndPrintf("\nTarget cluster context: %s\n", clusterCtxStr)
125
	common.LogAndPrintf("Running with the following config: \n\n%s\n\n", config)
126

127
	restConfig, clientset, err := kubeclient.New(config.KubeConfigPath, config.Context)
128
	if err != nil {
129
		return fmt.Errorf("could not initialize k8s client: %s ", err)
130
	}
131
	client, err := kube.NewCLIClient(kube.NewClientConfigForRestConfig(restConfig), "")
132
	if err != nil {
133
		return err
134
	}
135
	common.LogAndPrintf("\nCluster endpoint: %s\n", client.RESTConfig().Host)
136
	runner.SetClient(client)
137

138
	clusterResourcesCtx, getClusterResourcesCancel := context.WithTimeout(context.Background(), commandTimeout)
139
	curTime := time.Now()
140
	defer func() {
141
		if time.Until(curTime.Add(commandTimeout)) < 0 {
142
			message := "Timeout when running bug report command, please using --include or --exclude to filter"
143
			common.LogAndPrintf(message)
144
		}
145
		getClusterResourcesCancel()
146
	}()
147
	resources, err := cluster2.GetClusterResources(clusterResourcesCtx, clientset, config)
148
	if err != nil {
149
		return err
150
	}
151
	logRuntime(curTime, "Done collecting cluster resource")
152

153
	dumpRevisionsAndVersions(ctx, resources, config.IstioNamespace, config.DryRun)
154

155
	log.Infof("Cluster resource tree:\n\n%s\n\n", resources)
156
	paths, err := filter.GetMatchingPaths(config, resources)
157
	if err != nil {
158
		return err
159
	}
160

161
	common.LogAndPrintf("\n\nFetching logs for the following containers:\n\n%s\n", strings.Join(paths, "\n"))
162

163
	gatherInfo(runner, config, resources, paths)
164
	if len(gErrors) != 0 {
165
		log.Error(gErrors.ToError())
166
	}
167

168
	// TODO: sort by importance and discard any over the size limit.
169
	for path, text := range logs {
170
		namespace, _, pod, _, err := cluster2.ParsePath(path)
171
		if err != nil {
172
			log.Errorf(err.Error())
173
			continue
174
		}
175
		writeFile(filepath.Join(archive.ProxyOutputPath(tempDir, namespace, pod), common.ProxyContainerName+".log"), text, config.DryRun)
176
	}
177

178
	logRuntime(curTime, "Done with bug-report command before generating the archive file")
179

180
	outDir, err := os.Getwd()
181
	if err != nil {
182
		log.Errorf("using ./ to write archive: %s", err.Error())
183
		outDir = "."
184
	}
185
	if outputDir != "" {
186
		outDir = outputDir
187
	}
188
	outPath := filepath.Join(outDir, "bug-report.tar.gz")
189

190
	if !config.DryRun {
191
		common.LogAndPrintf("Creating an archive at %s.\n", outPath)
192
		archiveDir := archive.DirToArchive(tempDir)
193
		if tempDir != "" {
194
			archiveDir = tempDir
195
		}
196
		curTime = time.Now()
197
		err := archive.Create(archiveDir, outPath)
198
		fmt.Printf("Time used for creating the tar file is %v.\n", time.Since(curTime))
199
		if err != nil {
200
			return err
201
		}
202
		common.LogAndPrintf("Cleaning up temporary files in %s.\n", archiveDir)
203
		if err := os.RemoveAll(archiveDir); err != nil {
204
			return err
205
		}
206
	} else {
207
		common.LogAndPrintf("Dry run, skipping archive creation at %s.\n", outPath)
208
	}
209
	common.LogAndPrintf("Done.\n")
210
	return nil
211
}
212

213
func dumpRevisionsAndVersions(ctx cli.Context, resources *cluster2.Resources, istioNamespace string, dryRun bool) {
214
	defer logRuntime(time.Now(), "Done getting control plane revisions/versions")
215

216
	text := ""
217
	text += fmt.Sprintf("CLI version:\n%s\n\n", version.Info.LongForm())
218

219
	revisions := getIstioRevisions(resources)
220
	istioVersions, proxyVersions := getIstioVersions(ctx, istioNamespace, revisions)
221
	text += "The following Istio control plane revisions/versions were found in the cluster:\n"
222
	for rev, ver := range istioVersions {
223
		text += fmt.Sprintf("Revision %s:\n%s\n\n", rev, ver)
224
	}
225
	text += "The following proxy revisions/versions were found in the cluster:\n"
226
	for rev, ver := range proxyVersions {
227
		text += fmt.Sprintf("Revision %s: Versions {%s}\n", rev, strings.Join(ver, ", "))
228
	}
229
	common.LogAndPrintf(text)
230
	writeFile(filepath.Join(archive.OutputRootDir(tempDir), "versions"), text, dryRun)
231
}
232

233
// getIstioRevisions returns a slice with all Istio revisions detected in the cluster.
234
func getIstioRevisions(resources *cluster2.Resources) []string {
235
	revMap := sets.New[string]()
236
	for _, podLabels := range resources.Labels {
237
		for label, value := range podLabels {
238
			if label == label2.IoIstioRev.Name {
239
				revMap.Insert(value)
240
			}
241
		}
242
	}
243
	for _, podAnnotations := range resources.Annotations {
244
		for annotation, value := range podAnnotations {
245
			if annotation == label2.IoIstioRev.Name {
246
				revMap.Insert(value)
247
			}
248
		}
249
	}
250
	return sets.SortedList(revMap)
251
}
252

253
// getIstioVersions returns a mapping of revision to aggregated version string for Istio components and revision to
254
// slice of versions for proxies. Any errors are embedded in the revision strings.
255
func getIstioVersions(ctx cli.Context, istioNamespace string, revisions []string) (map[string]string, map[string][]string) {
256
	istioVersions := make(map[string]string)
257
	proxyVersionsMap := make(map[string]sets.String)
258
	proxyVersions := make(map[string][]string)
259
	for _, revision := range revisions {
260
		client, err := ctx.CLIClientWithRevision(revision)
261
		if err != nil {
262
			log.Error(err)
263
			continue
264
		}
265
		istioVersions[revision] = getIstioVersion(client, istioNamespace)
266
		proxyInfo, err := proxy.GetProxyInfo(client, istioNamespace)
267
		if err != nil {
268
			log.Error(err)
269
			continue
270
		}
271
		for _, pi := range *proxyInfo {
272
			sets.InsertOrNew(proxyVersionsMap, revision, pi.IstioVersion)
273
		}
274
	}
275
	for revision, vmap := range proxyVersionsMap {
276
		for v := range vmap {
277
			proxyVersions[revision] = append(proxyVersions[revision], v)
278
		}
279
	}
280
	return istioVersions, proxyVersions
281
}
282

283
func getIstioVersion(kubeClient kube.CLIClient, istioNamespace string) string {
284
	versions, err := kubeClient.GetIstioVersions(context.TODO(), istioNamespace)
285
	if err != nil {
286
		return err.Error()
287
	}
288
	return pretty.Sprint(versions)
289
}
290

291
// gatherInfo fetches all logs, resources, debug etc. using goroutines.
292
// proxy logs and info are saved in logs/stats/importance global maps.
293
// Errors are reported through gErrors.
294
func gatherInfo(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources, paths []string) {
295
	// no timeout on mandatoryWg.
296
	var mandatoryWg sync.WaitGroup
297
	cmdTimer := time.NewTimer(time.Duration(config.CommandTimeout))
298
	beginTime := time.Now()
299

300
	client, err := kube.NewCLIClient(kube.BuildClientCmd(config.KubeConfigPath, config.Context), "")
301
	if err != nil {
302
		appendGlobalErr(err)
303
	}
304

305
	clusterDir := archive.ClusterInfoPath(tempDir)
306

307
	params := &content.Params{
308
		Runner:      runner,
309
		DryRun:      config.DryRun,
310
		KubeConfig:  config.KubeConfigPath,
311
		KubeContext: config.Context,
312
	}
313
	common.LogAndPrintf("\nFetching Istio control plane information from cluster.\n\n")
314
	getFromCluster(content.GetK8sResources, params, clusterDir, &mandatoryWg)
315
	getFromCluster(content.GetCRs, params, clusterDir, &mandatoryWg)
316
	getFromCluster(content.GetEvents, params, clusterDir, &mandatoryWg)
317
	getFromCluster(content.GetClusterInfo, params, clusterDir, &mandatoryWg)
318
	getFromCluster(content.GetNodeInfo, params, clusterDir, &mandatoryWg)
319
	getFromCluster(content.GetSecrets, params.SetVerbose(config.FullSecrets), clusterDir, &mandatoryWg)
320
	getFromCluster(content.GetPodInfo, params.SetIstioNamespace(config.IstioNamespace), clusterDir, &mandatoryWg)
321

322
	common.LogAndPrintf("\nFetching CNI logs from cluster.\n\n")
323
	for _, cniPod := range resources.CniPod {
324
		getCniLogs(runner, config, resources, cniPod.Namespace, cniPod.Name, &mandatoryWg)
325
	}
326

327
	// optionalWg is subject to timer.
328
	var optionalWg sync.WaitGroup
329
	for _, p := range paths {
330
		namespace, _, pod, container, err := cluster2.ParsePath(p)
331
		if err != nil {
332
			log.Error(err.Error())
333
			continue
334
		}
335

336
		cp := params.SetNamespace(namespace).SetPod(pod).SetContainer(container)
337
		proxyDir := archive.ProxyOutputPath(tempDir, namespace, pod)
338
		switch {
339
		case common.IsProxyContainer(params.ClusterVersion, container):
340
			if !ambient.IsZtunnelPod(client, pod, namespace) {
341
				getFromCluster(content.GetCoredumps, cp, filepath.Join(proxyDir, "cores"), &mandatoryWg)
342
				getFromCluster(content.GetNetstat, cp, proxyDir, &mandatoryWg)
343
				getFromCluster(content.GetProxyInfo, cp, archive.ProxyOutputPath(tempDir, namespace, pod), &optionalWg)
344
				getProxyLogs(runner, config, resources, p, namespace, pod, container, &optionalWg)
345
			} else {
346
				getFromCluster(content.GetNetstat, cp, proxyDir, &mandatoryWg)
347
				getFromCluster(content.GetZtunnelInfo, cp, archive.ProxyOutputPath(tempDir, namespace, pod), &optionalWg)
348
				getProxyLogs(runner, config, resources, p, namespace, pod, container, &optionalWg)
349
			}
350
		case resources.IsDiscoveryContainer(params.ClusterVersion, namespace, pod, container):
351
			getFromCluster(content.GetIstiodInfo, cp, archive.IstiodPath(tempDir, namespace, pod), &mandatoryWg)
352
			getIstiodLogs(runner, config, resources, namespace, pod, &mandatoryWg)
353

354
		case common.IsOperatorContainer(params.ClusterVersion, container):
355
			getOperatorLogs(runner, config, resources, namespace, pod, &optionalWg)
356
		}
357
	}
358

359
	// Not all items are subject to timeout. Proceed only if the non-cancellable items have completed.
360
	mandatoryWg.Wait()
361

362
	// If log fetches have completed, cancel the timeout.
363
	go func() {
364
		optionalWg.Wait()
365
		cmdTimer.Reset(0)
366
	}()
367

368
	// Wait for log fetches, up to the timeout.
369
	<-cmdTimer.C
370

371
	// Find the timeout duration left for the analysis process.
372
	analyzeTimeout := time.Until(beginTime.Add(time.Duration(config.CommandTimeout)))
373

374
	// Analyze runs many queries internally, so run these queries sequentially and after everything else has finished.
375
	runAnalyze(config, params, analyzeTimeout)
376
}
377

378
// getFromCluster runs a cluster info fetching function f against the cluster and writes the results to fileName.
379
// Runs if a goroutine, with errors reported through gErrors.
380
func getFromCluster(f func(params *content.Params) (map[string]string, error), params *content.Params, dir string, wg *sync.WaitGroup) {
381
	startTime := time.Now()
382
	wg.Add(1)
383
	log.Infof("Waiting on %s", runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
384
	go func() {
385
		defer func() {
386
			wg.Done()
387
			logRuntime(startTime, "Done getting from cluster for %v", runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
388
		}()
389

390
		out, err := f(params)
391
		appendGlobalErr(err)
392
		if err == nil {
393
			writeFiles(dir, out, params.DryRun)
394
		}
395
		log.Infof("Done with %s", runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
396
	}()
397
}
398

399
// getProxyLogs fetches proxy logs for the given namespace/pod/container and stores the output in global structs.
400
// Runs if a goroutine, with errors reported through gErrors.
401
// TODO(stewartbutler): output the logs to a more robust/complete structure.
402
func getProxyLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources,
403
	path, namespace, pod, container string, wg *sync.WaitGroup,
404
) {
405
	startTime := time.Now()
406
	wg.Add(1)
407
	log.Infof("Waiting on proxy logs %v/%v/%v", namespace, pod, container)
408
	go func() {
409
		defer func() {
410
			wg.Done()
411
			logRuntime(startTime, "Done getting from proxy logs for %v/%v/%v", namespace, pod, container)
412
		}()
413

414
		clog, cstat, imp, err := getLog(runner, resources, config, namespace, pod, container)
415
		appendGlobalErr(err)
416
		lock.Lock()
417
		if err == nil {
418
			logs[path], stats[path], importance[path] = clog, cstat, imp
419
		}
420
		lock.Unlock()
421
		log.Infof("Done with proxy logs %v/%v/%v", namespace, pod, container)
422
	}()
423
}
424

425
// getIstiodLogs fetches Istiod logs for the given namespace/pod and writes the output.
426
// Runs if a goroutine, with errors reported through gErrors.
427
func getIstiodLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources,
428
	namespace, pod string, wg *sync.WaitGroup,
429
) {
430
	startTime := time.Now()
431
	wg.Add(1)
432
	log.Infof("Waiting on Istiod logs for %v/%v", namespace, pod)
433
	go func() {
434
		defer func() {
435
			wg.Done()
436
			logRuntime(startTime, "Done getting Istiod logs for %v/%v", namespace, pod)
437
		}()
438

439
		clog, _, _, err := getLog(runner, resources, config, namespace, pod, common.DiscoveryContainerName)
440
		appendGlobalErr(err)
441
		writeFile(filepath.Join(archive.IstiodPath(tempDir, namespace, pod), "discovery.log"), clog, config.DryRun)
442
		log.Infof("Done with Istiod logs for %v/%v", namespace, pod)
443
	}()
444
}
445

446
// getOperatorLogs fetches istio-operator logs for the given namespace/pod and writes the output.
447
func getOperatorLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources,
448
	namespace, pod string, wg *sync.WaitGroup,
449
) {
450
	startTime := time.Now()
451
	wg.Add(1)
452
	log.Infof("Waiting on operator logs for %v/%v", namespace, pod)
453
	go func() {
454
		defer func() {
455
			wg.Done()
456
			logRuntime(startTime, "Done getting operator logs for %v/%v", namespace, pod)
457
		}()
458

459
		clog, _, _, err := getLog(runner, resources, config, namespace, pod, common.OperatorContainerName)
460
		appendGlobalErr(err)
461
		writeFile(filepath.Join(archive.OperatorPath(tempDir, namespace, pod), "operator.log"), clog, config.DryRun)
462
		log.Infof("Done with operator logs for %v/%v", namespace, pod)
463
	}()
464
}
465

466
// getCniLogs fetches Cni logs from istio-cni-node daemonsets inside namespace kube-system and writes the output
467
// Runs if a goroutine, with errors reported through gErrors
468
func getCniLogs(runner *kubectlcmd.Runner, config *config.BugReportConfig, resources *cluster2.Resources,
469
	namespace, pod string, wg *sync.WaitGroup,
470
) {
471
	startTime := time.Now()
472
	wg.Add(1)
473
	log.Infof("Waiting on CNI logs for %v", pod)
474
	go func() {
475
		defer func() {
476
			wg.Done()
477
			logRuntime(startTime, "Done getting CNI logs for %v", pod)
478
		}()
479

480
		clog, _, _, err := getLog(runner, resources, config, namespace, pod, "")
481
		appendGlobalErr(err)
482
		writeFile(filepath.Join(archive.CniPath(tempDir, pod), "cni.log"), clog, config.DryRun)
483
		log.Infof("Done with CNI logs %v", pod)
484
	}()
485
}
486

487
// getLog fetches the logs for the given namespace/pod/container and returns the log text and stats for it.
488
func getLog(runner *kubectlcmd.Runner, resources *cluster2.Resources, config *config.BugReportConfig,
489
	namespace, pod, container string,
490
) (string, *processlog.Stats, int, error) {
491
	defer logRuntime(time.Now(), "Done getting logs only for %v/%v/%v", namespace, pod, container)
492

493
	log.Infof("Getting logs for %s/%s/%s...", namespace, pod, container)
494
	clog, err := runner.Logs(namespace, pod, container, false, config.DryRun)
495
	if err != nil {
496
		return "", nil, 0, err
497
	}
498
	if resources.ContainerRestarts(namespace, pod, container, common.IsCniPod(pod)) > 0 {
499
		pclog, err := runner.Logs(namespace, pod, container, true, config.DryRun)
500
		if err != nil {
501
			return "", nil, 0, err
502
		}
503
		clog = "========= Previous log present (appended at the end) =========\n\n" + clog +
504
			"\n\n========= Previous log =========\n\n" + pclog
505
	}
506
	var cstat *processlog.Stats
507
	clog, cstat = processlog.Process(config, clog)
508
	return clog, cstat, cstat.Importance(), nil
509
}
510

511
func runAnalyze(config *config.BugReportConfig, params *content.Params, analyzeTimeout time.Duration) {
512
	newParam := params.SetNamespace(common.NamespaceAll)
513

514
	defer logRuntime(time.Now(), "Done running Istio analyze on all namespaces and report")
515

516
	common.LogAndPrintf("Running Istio analyze on all namespaces and report as below:")
517
	out, err := content.GetAnalyze(newParam.SetIstioNamespace(config.IstioNamespace), analyzeTimeout)
518
	if err != nil {
519
		log.Error(err.Error())
520
		return
521
	}
522
	common.LogAndPrintf("\nAnalysis Report:\n")
523
	common.LogAndPrintf(out[common.StrNamespaceAll])
524
	common.LogAndPrintf("\n")
525
	writeFiles(archive.AnalyzePath(tempDir, common.StrNamespaceAll), out, config.DryRun)
526
}
527

528
func writeFiles(dir string, files map[string]string, dryRun bool) {
529
	defer logRuntime(time.Now(), "Done writing files for dir %v", dir)
530
	for fname, text := range files {
531
		writeFile(filepath.Join(dir, fname), text, dryRun)
532
	}
533
}
534

535
func writeFile(path, text string, dryRun bool) {
536
	if dryRun {
537
		return
538
	}
539
	if strings.TrimSpace(text) == "" {
540
		return
541
	}
542
	mkdirOrExit(path)
543

544
	defer logRuntime(time.Now(), "Done writing file for path %v", path)
545

546
	if err := os.WriteFile(path, []byte(text), 0o644); err != nil {
547
		log.Errorf(err.Error())
548
	}
549
}
550

551
func mkdirOrExit(fpath string) {
552
	if err := os.MkdirAll(path.Dir(fpath), 0o755); err != nil {
553
		fmt.Printf("Could not create output directories: %s", err)
554
		os.Exit(-1)
555
	}
556
}
557

558
func appendGlobalErr(err error) {
559
	if err == nil {
560
		return
561
	}
562
	lock.Lock()
563
	gErrors = util.AppendErr(gErrors, err)
564
	lock.Unlock()
565
}
566

567
func configLogs(opt *log.Options) error {
568
	logDir := filepath.Join(archive.OutputRootDir(tempDir), "bug-report.log")
569
	mkdirOrExit(logDir)
570
	f, err := os.Create(logDir)
571
	if err != nil {
572
		return err
573
	}
574
	f.Close()
575
	op := []string{logDir}
576
	opt2 := *opt
577
	opt2.OutputPaths = op
578
	opt2.ErrorOutputPaths = op
579
	opt2.SetDefaultOutputLevel("default", log.InfoLevel)
580

581
	return log.Configure(&opt2)
582
}
583

584
func logRuntime(start time.Time, format string, args ...any) {
585
	log.WithLabels("runtime", time.Since(start)).Infof(format, args...)
586
}
587