istio

1
// Copyright Istio Authors
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14

15
package processlog
16

17
import (
18
	"encoding/json"
19
	"regexp"
20
	"strings"
21
	"time"
22

23
	"istio.io/istio/tools/bug-report/pkg/config"
24
	"istio.io/istio/tools/bug-report/pkg/util/match"
25
)
26

27
const (
28
	levelFatal = "fatal"
29
	levelError = "error"
30
	levelWarn  = "warn"
31
	levelInfo  = "info"
32
	levelDebug = "debug"
33
	levelTrace = "trace"
34
)
35

36
var ztunnelLogPattern = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+(?:\w+\s+)?(\w+)\s+([\w\.:]+)(.*)`)
37

38
// Stats represents log statistics.
39
type Stats struct {
40
	numFatals   int
41
	numErrors   int
42
	numWarnings int
43
}
44

45
// Importance returns an integer that indicates the importance of the log, based on the given Stats in s.
46
// Larger numbers are more important.
47
func (s *Stats) Importance() int {
48
	if s == nil {
49
		return 0
50
	}
51
	return 1000*s.numFatals + 100*s.numErrors + 10*s.numWarnings
52
}
53

54
// Process processes logStr based on the supplied config and returns the processed log along with statistics on it.
55
func Process(config *config.BugReportConfig, logStr string) (string, *Stats) {
56
	if !config.TimeFilterApplied {
57
		return logStr, getStats(config, logStr)
58
	}
59
	out := getTimeRange(logStr, config.StartTime, config.EndTime)
60
	return out, getStats(config, out)
61
}
62

63
// getTimeRange returns the log lines that fall inside the start to end time range, inclusive.
64
func getTimeRange(logStr string, start, end time.Time) string {
65
	var sb strings.Builder
66
	write := false
67
	for _, l := range strings.Split(logStr, "\n") {
68
		t, _, _, valid := parseLog(l)
69
		if valid {
70
			write = false
71
			if (t.Equal(start) || t.After(start)) && (t.Equal(end) || t.Before(end)) {
72
				write = true
73
			}
74
		}
75
		if write {
76
			sb.WriteString(l)
77
			sb.WriteString("\n")
78
		}
79
	}
80

81
	return sb.String()
82
}
83

84
// getStats returns statistics for the given log string.
85
func getStats(config *config.BugReportConfig, logStr string) *Stats {
86
	out := &Stats{}
87
	for _, l := range strings.Split(logStr, "\n") {
88
		_, level, text, valid := parseLog(l)
89
		if !valid {
90
			continue
91
		}
92
		switch level {
93
		case levelFatal, levelError, levelWarn:
94
			if match.MatchesGlobs(text, config.IgnoredErrors) {
95
				continue
96
			}
97
			switch level {
98
			case levelFatal:
99
				out.numFatals++
100
			case levelError:
101
				out.numErrors++
102
			case levelWarn:
103
				out.numWarnings++
104
			}
105
		default:
106
		}
107
	}
108
	return out
109
}
110

111
func parseLog(line string) (timeStamp *time.Time, level string, text string, valid bool) {
112
	if isJSONLog(line) {
113
		return parseJSONLog(line)
114
	}
115
	return processPlainLog(line)
116
}
117

118
func processPlainLog(line string) (timeStamp *time.Time, level string, text string, valid bool) {
119
	lv := strings.Split(line, "\t")
120
	if len(lv) < 3 {
121
		// maybe ztunnel logs
122
		// TODO remove this when https://github.com/istio/ztunnel/issues/453 is fixed
123
		matches := ztunnelLogPattern.FindStringSubmatch(line)
124
		if len(matches) < 5 {
125
			return nil, "", "", false
126
		}
127
		lv = matches[1:]
128
	}
129
	ts, err := time.Parse(time.RFC3339Nano, lv[0])
130
	if err != nil {
131
		return nil, "", "", false
132
	}
133
	timeStamp = &ts
134
	switch strings.ToLower(lv[1]) {
135
	case levelFatal, levelError, levelWarn, levelInfo, levelDebug, levelTrace:
136
		level = lv[1]
137
	default:
138
		return nil, "", "", false
139
	}
140
	text = strings.Join(lv[2:], "\t")
141
	valid = true
142
	return
143
}
144

145
type logJSON struct {
146
	Time  string
147
	Level string
148
	Msg   string
149
}
150

151
func parseJSONLog(line string) (timeStamp *time.Time, level string, text string, valid bool) {
152
	lj := logJSON{}
153

154
	err := json.Unmarshal([]byte(line), &lj)
155
	if err != nil {
156
		return nil, "", "", false
157
	}
158

159
	// todo: add logging for err
160
	m := lj.Msg
161
	if m == "" {
162
		return nil, "", "", false
163
	}
164

165
	t := lj.Time
166
	ts, err := time.Parse(time.RFC3339Nano, t)
167
	if err != nil {
168
		return nil, "", "", false
169
	}
170

171
	l := lj.Level
172
	switch l {
173
	case levelFatal, levelError, levelWarn, levelInfo, levelDebug, levelTrace:
174
	default:
175
		return nil, "", "", false
176
	}
177
	return &ts, l, m, true
178
}
179

180
func isJSONLog(logStr string) bool {
181
	return strings.HasPrefix(logStr, "{")
182
}
183