1
// Copyright 2015 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
6
Package trace implements tracing of requests and long-lived objects.
7
It exports HTTP interfaces on /debug/requests and /debug/events.
9
A trace.Trace provides tracing for short-lived objects, usually requests.
10
A request handler might be implemented like this:
12
func fooHandler(w http.ResponseWriter, req *http.Request) {
13
tr := trace.New("mypkg.Foo", req.URL.Path)
16
tr.LazyPrintf("some event %q happened", str)
18
if err := somethingImportant(); err != nil {
19
tr.LazyPrintf("somethingImportant failed: %v", err)
24
The /debug/requests HTTP endpoint organizes the traces by family,
25
errors, and duration. It also provides histogram of request duration
28
A trace.EventLog provides tracing for long-lived objects, such as RPC
31
// A Fetcher fetches URL paths for a single domain.
37
func NewFetcher(domain string) *Fetcher {
40
trace.NewEventLog("mypkg.Fetcher", domain),
44
func (f *Fetcher) Fetch(path string) (string, error) {
45
resp, err := http.Get("http://" + f.domain + "/" + path)
47
f.events.Errorf("Get(%q) = %v", path, err)
50
f.events.Printf("Get(%q) = %s", path, resp.Status)
54
func (f *Fetcher) Close() error {
59
The /debug/events HTTP endpoint organizes the event logs by family and
60
by time since the last error. The expanded view displays recent log
61
entries and the log's call stack.
63
package trace // import "golang.org/x/net/trace"
82
"golang.org/x/net/internal/timeseries"
85
// DebugUseAfterFinish controls whether to debug uses of Trace values after finishing.
86
// FOR DEBUGGING ONLY. This will slow down the program.
87
var DebugUseAfterFinish = false
89
// HTTP ServeMux paths.
91
debugRequestsPath = "/debug/requests"
92
debugEventsPath = "/debug/events"
95
// AuthRequest determines whether a specific request is permitted to load the
96
// /debug/requests or /debug/events pages.
98
// It returns two bools; the first indicates whether the page may be viewed at all,
99
// and the second indicates whether sensitive events will be shown.
101
// AuthRequest may be replaced by a program to customize its authorization requirements.
103
// The default AuthRequest function returns (true, true) if and only if the request
104
// comes from localhost/127.0.0.1/[::1].
105
var AuthRequest = func(req *http.Request) (any, sensitive bool) {
106
// RemoteAddr is commonly in the form "IP" or "IP:port".
107
// If it is in the form "IP:port", split off the port.
108
host, _, err := net.SplitHostPort(req.RemoteAddr)
110
host = req.RemoteAddr
113
case "localhost", "127.0.0.1", "::1":
121
_, pat := http.DefaultServeMux.Handler(&http.Request{URL: &url.URL{Path: debugRequestsPath}})
122
if pat == debugRequestsPath {
123
panic("/debug/requests is already registered. You may have two independent copies of " +
124
"golang.org/x/net/trace in your binary, trying to maintain separate state. This may " +
125
"involve a vendored copy of golang.org/x/net/trace.")
128
// TODO(jbd): Serve Traces from /debug/traces in the future?
129
// There is no requirement for a request to be present to have traces.
130
http.HandleFunc(debugRequestsPath, Traces)
131
http.HandleFunc(debugEventsPath, Events)
134
// NewContext returns a copy of the parent context
135
// and associates it with a Trace.
136
func NewContext(ctx context.Context, tr Trace) context.Context {
137
return context.WithValue(ctx, contextKey, tr)
140
// FromContext returns the Trace bound to the context, if any.
141
func FromContext(ctx context.Context) (tr Trace, ok bool) {
142
tr, ok = ctx.Value(contextKey).(Trace)
146
// Traces responds with traces from the program.
147
// The package initialization registers it in http.DefaultServeMux
148
// at /debug/requests.
150
// It performs authorization by running AuthRequest.
151
func Traces(w http.ResponseWriter, req *http.Request) {
152
any, sensitive := AuthRequest(req)
154
http.Error(w, "not allowed", http.StatusUnauthorized)
157
w.Header().Set("Content-Type", "text/html; charset=utf-8")
158
Render(w, req, sensitive)
161
// Events responds with a page of events collected by EventLogs.
162
// The package initialization registers it in http.DefaultServeMux
165
// It performs authorization by running AuthRequest.
166
func Events(w http.ResponseWriter, req *http.Request) {
167
any, sensitive := AuthRequest(req)
169
http.Error(w, "not allowed", http.StatusUnauthorized)
172
w.Header().Set("Content-Type", "text/html; charset=utf-8")
173
RenderEvents(w, req, sensitive)
176
// Render renders the HTML page typically served at /debug/requests.
177
// It does not do any auth checking. The request may be nil.
179
// Most users will use the Traces handler.
180
func Render(w io.Writer, req *http.Request, sensitive bool) {
183
ActiveTraceCount map[string]int
184
CompletedTraces map[string]*family
186
// Set when a bucket has been selected.
193
ShowSensitive bool // whether to show sensitive events
195
Histogram template.HTML
196
HistogramWindow string // e.g. "last minute", "last hour", "all time"
198
// If non-zero, the set of traces is a partial set,
199
// and this is the total number.
202
CompletedTraces: completedTraces,
205
data.ShowSensitive = sensitive
207
// Allow show_sensitive=0 to force hiding of sensitive data for testing.
208
// This only goes one way; you can't use show_sensitive=1 to see things.
209
if req.FormValue("show_sensitive") == "0" {
210
data.ShowSensitive = false
213
if exp, err := strconv.ParseBool(req.FormValue("exp")); err == nil {
216
if exp, err := strconv.ParseBool(req.FormValue("rtraced")); err == nil {
222
data.Families = make([]string, 0, len(completedTraces))
223
for fam := range completedTraces {
224
data.Families = append(data.Families, fam)
226
completedMu.RUnlock()
227
sort.Strings(data.Families)
229
// We are careful here to minimize the time spent locking activeMu,
230
// since that lock is required every time an RPC starts and finishes.
231
data.ActiveTraceCount = make(map[string]int, len(data.Families))
233
for fam, s := range activeTraces {
234
data.ActiveTraceCount[fam] = s.Len()
239
data.Family, data.Bucket, ok = parseArgs(req)
243
case data.Bucket == -1:
245
n := data.ActiveTraceCount[data.Family]
246
data.Traces = getActiveTraces(data.Family)
247
if len(data.Traces) < n {
250
case data.Bucket < bucketsPerFamily:
251
if b := lookupBucket(data.Family, data.Bucket); b != nil {
252
data.Traces = b.Copy(data.Traced)
255
if f := getFamily(data.Family, false); f != nil {
256
var obs timeseries.Observable
258
switch o := data.Bucket - bucketsPerFamily; o {
260
obs = f.Latency.Minute()
261
data.HistogramWindow = "last minute"
263
obs = f.Latency.Hour()
264
data.HistogramWindow = "last hour"
266
obs = f.Latency.Total()
267
data.HistogramWindow = "all time"
269
f.LatencyMu.RUnlock()
271
data.Histogram = obs.(*histogram).html()
276
if data.Traces != nil {
277
defer data.Traces.Free()
278
sort.Sort(data.Traces)
282
defer completedMu.RUnlock()
283
if err := pageTmpl().ExecuteTemplate(w, "Page", data); err != nil {
284
log.Printf("net/trace: Failed executing template: %v", err)
288
func parseArgs(req *http.Request) (fam string, b int, ok bool) {
292
fam, bStr := req.FormValue("fam"), req.FormValue("b")
293
if fam == "" || bStr == "" {
296
b, err := strconv.Atoi(bStr)
297
if err != nil || b < -1 {
304
func lookupBucket(fam string, b int) *traceBucket {
305
f := getFamily(fam, false)
306
if f == nil || b < 0 || b >= len(f.Buckets) {
312
type contextKeyT string
314
var contextKey = contextKeyT("golang.org/x/net/trace.Trace")
316
// Trace represents an active request.
317
type Trace interface {
318
// LazyLog adds x to the event log. It will be evaluated each time the
319
// /debug/requests page is rendered. Any memory referenced by x will be
320
// pinned until the trace is finished and later discarded.
321
LazyLog(x fmt.Stringer, sensitive bool)
323
// LazyPrintf evaluates its arguments with fmt.Sprintf each time the
324
// /debug/requests page is rendered. Any memory referenced by a will be
325
// pinned until the trace is finished and later discarded.
326
LazyPrintf(format string, a ...interface{})
328
// SetError declares that this trace resulted in an error.
331
// SetRecycler sets a recycler for the trace.
332
// f will be called for each event passed to LazyLog at a time when
333
// it is no longer required, whether while the trace is still active
334
// and the event is discarded, or when a completed trace is discarded.
335
SetRecycler(f func(interface{}))
337
// SetTraceInfo sets the trace info for the trace.
338
// This is currently unused.
339
SetTraceInfo(traceID, spanID uint64)
341
// SetMaxEvents sets the maximum number of events that will be stored
342
// in the trace. This has no effect if any events have already been
343
// added to the trace.
346
// Finish declares that this trace is complete.
347
// The trace should not be used after calling this method.
351
type lazySprintf struct {
356
func (l *lazySprintf) String() string {
357
return fmt.Sprintf(l.format, l.a...)
360
// New returns a new Trace with the specified family and title.
361
func New(family, title string) Trace {
364
tr.Family, tr.Title = family, title
365
tr.Start = time.Now()
366
tr.maxEvents = maxEventsPerTrace
367
tr.events = tr.eventsBuf[:0]
370
s := activeTraces[tr.Family]
374
s = activeTraces[tr.Family] // check again
377
activeTraces[tr.Family] = s
383
// Trigger allocation of the completed trace structure for this family.
384
// This will cause the family to be present in the request page during
385
// the first trace of this family. We don't care about the return value,
386
// nor is there any need for this to run inline, so we execute it in its
387
// own goroutine, but only if the family isn't allocated yet.
389
if _, ok := completedTraces[tr.Family]; !ok {
390
go allocFamily(tr.Family)
392
completedMu.RUnlock()
397
func (tr *trace) Finish() {
398
elapsed := time.Since(tr.Start)
403
if DebugUseAfterFinish {
404
buf := make([]byte, 4<<10) // 4 KB should be enough
405
n := runtime.Stack(buf, false)
406
tr.finishStack = buf[:n]
410
m := activeTraces[tr.Family]
414
f := getFamily(tr.Family, true)
415
tr.mu.RLock() // protects tr fields in Cond.match calls
416
for _, b := range f.Buckets {
417
if b.Cond.match(tr) {
423
// Add a sample of elapsed time as microseconds to the family's timeseries
425
h.addMeasurement(elapsed.Nanoseconds() / 1e3)
430
tr.unref() // matches ref in New
436
maxActiveTraces = 20 // Maximum number of active traces to show.
437
maxEventsPerTrace = 10
438
numHistogramBuckets = 38
442
// The active traces.
443
activeMu sync.RWMutex
444
activeTraces = make(map[string]*traceSet) // family -> traces
446
// Families of completed traces.
447
completedMu sync.RWMutex
448
completedTraces = make(map[string]*family) // family -> traces
451
type traceSet struct {
455
// We could avoid the entire map scan in FirstN by having a slice of all the traces
456
// ordered by start time, and an index into that from the trace struct, with a periodic
457
// repack of the slice after enough traces finish; we could also use a skip list or similar.
458
// However, that would shift some of the expense from /debug/requests time to RPC time,
459
// which is probably the wrong trade-off.
462
func (ts *traceSet) Len() int {
464
defer ts.mu.RUnlock()
468
func (ts *traceSet) Add(tr *trace) {
471
ts.m = make(map[*trace]bool)
477
func (ts *traceSet) Remove(tr *trace) {
483
// FirstN returns the first n traces ordered by time.
484
func (ts *traceSet) FirstN(n int) traceList {
486
defer ts.mu.RUnlock()
491
trl := make(traceList, 0, n)
493
// Fast path for when no selectivity is needed.
495
for tr := range ts.m {
497
trl = append(trl, tr)
503
// Pick the oldest n traces.
504
// This is inefficient. See the comment in the traceSet struct.
505
for tr := range ts.m {
506
// Put the first n traces into trl in the order they occur.
507
// When we have n, sort trl, and thereafter maintain its order.
510
trl = append(trl, tr)
512
// This is guaranteed to happen exactly once during this loop.
517
if tr.Start.After(trl[n-1].Start) {
521
// Find where to insert this one.
523
i := sort.Search(n, func(i int) bool { return trl[i].Start.After(tr.Start) })
525
copy(trl[i+1:], trl[i:])
532
func getActiveTraces(fam string) traceList {
534
s := activeTraces[fam]
539
return s.FirstN(maxActiveTraces)
542
func getFamily(fam string, allocNew bool) *family {
544
f := completedTraces[fam]
545
completedMu.RUnlock()
546
if f == nil && allocNew {
552
func allocFamily(fam string) *family {
554
defer completedMu.Unlock()
555
f := completedTraces[fam]
558
completedTraces[fam] = f
563
// family represents a set of trace buckets and associated latency information.
565
// traces may occur in multiple buckets.
566
Buckets [bucketsPerFamily]*traceBucket
568
// latency time series
569
LatencyMu sync.RWMutex
570
Latency *timeseries.MinuteHourSeries
573
func newFamily() *family {
575
Buckets: [bucketsPerFamily]*traceBucket{
577
{Cond: minCond(50 * time.Millisecond)},
578
{Cond: minCond(100 * time.Millisecond)},
579
{Cond: minCond(200 * time.Millisecond)},
580
{Cond: minCond(500 * time.Millisecond)},
581
{Cond: minCond(1 * time.Second)},
582
{Cond: minCond(10 * time.Second)},
583
{Cond: minCond(100 * time.Second)},
586
Latency: timeseries.NewMinuteHourSeries(func() timeseries.Observable { return new(histogram) }),
590
// traceBucket represents a size-capped bucket of historic traces,
591
// along with a condition for a trace to belong to the bucket.
592
type traceBucket struct {
595
// Ring buffer implementation of a fixed-size FIFO queue.
597
buf [tracesPerBucket]*trace
598
start int // < tracesPerBucket
599
length int // <= tracesPerBucket
602
func (b *traceBucket) Add(tr *trace) {
606
i := b.start + b.length
607
if i >= tracesPerBucket {
610
if b.length == tracesPerBucket {
611
// "Remove" an element from the bucket.
614
if b.start == tracesPerBucket {
619
if b.length < tracesPerBucket {
625
// Copy returns a copy of the traces in the bucket.
626
// If tracedOnly is true, only the traces with trace information will be returned.
627
// The logs will be ref'd before returning; the caller should call
628
// the Free method when it is done with them.
629
// TODO(dsymonds): keep track of traced requests in separate buckets.
630
func (b *traceBucket) Copy(tracedOnly bool) traceList {
634
trl := make(traceList, 0, b.length)
635
for i, x := 0, b.start; i < b.length; i++ {
637
if !tracedOnly || tr.spanID != 0 {
639
trl = append(trl, tr)
649
func (b *traceBucket) Empty() bool {
655
// cond represents a condition on a trace.
661
type minCond time.Duration
663
func (m minCond) match(t *trace) bool { return t.Elapsed >= time.Duration(m) }
664
func (m minCond) String() string { return fmt.Sprintf("≥%gs", time.Duration(m).Seconds()) }
666
type errorCond struct{}
668
func (e errorCond) match(t *trace) bool { return t.IsError }
669
func (e errorCond) String() string { return "errors" }
671
type traceList []*trace
673
// Free calls unref on each element of the list.
674
func (trl traceList) Free() {
675
for _, t := range trl {
680
// traceList may be sorted in reverse chronological order.
681
func (trl traceList) Len() int { return len(trl) }
682
func (trl traceList) Less(i, j int) bool { return trl[i].Start.After(trl[j].Start) }
683
func (trl traceList) Swap(i, j int) { trl[i], trl[j] = trl[j], trl[i] }
685
// An event is a timestamped log entry in a trace.
688
Elapsed time.Duration // since previous event in trace
689
NewDay bool // whether this event is on a different day to the previous event
690
Recyclable bool // whether this event was passed via LazyLog
691
Sensitive bool // whether this event contains sensitive information
692
What interface{} // string or fmt.Stringer
695
// WhenString returns a string representation of the elapsed time of the event.
696
// It will include the date if midnight was crossed.
697
func (e event) WhenString() string {
699
return e.When.Format("2006/01/02 15:04:05.000000")
701
return e.When.Format("15:04:05.000000")
704
// discarded represents a number of discarded events.
705
// It is stored as *discarded to make it easier to update in-place.
708
func (d *discarded) String() string {
709
return fmt.Sprintf("(%d events discarded)", int(*d))
712
// trace represents an active or complete request,
713
// either sent or received by this program.
715
// Family is the top-level grouping of traces to which this belongs.
718
// Title is the title of this trace.
721
// Start time of the this trace.
725
events []event // Append-only sequence of events (modulo discards).
727
recycler func(interface{})
728
IsError bool // Whether this trace resulted in an error.
729
Elapsed time.Duration // Elapsed time for this trace, zero while active.
730
traceID uint64 // Trace information if non-zero.
733
refs int32 // how many buckets this is in
734
disc discarded // scratch space to avoid allocation
736
finishStack []byte // where finish was called, if DebugUseAfterFinish is set
738
eventsBuf [4]event // preallocated buffer in case we only log a few events
741
func (tr *trace) reset() {
742
// Clear all but the mutex. Mutexes may not be copied, even when unlocked.
745
tr.Start = time.Time{}
760
for i := range tr.eventsBuf {
761
tr.eventsBuf[i] = event{}
765
// delta returns the elapsed time since the last event or the trace start,
766
// and whether it spans midnight.
768
func (tr *trace) delta(t time.Time) (time.Duration, bool) {
769
if len(tr.events) == 0 {
770
return t.Sub(tr.Start), false
772
prev := tr.events[len(tr.events)-1].When
773
return t.Sub(prev), prev.Day() != t.Day()
776
func (tr *trace) addEvent(x interface{}, recyclable, sensitive bool) {
777
if DebugUseAfterFinish && tr.finishStack != nil {
778
buf := make([]byte, 4<<10) // 4 KB should be enough
779
n := runtime.Stack(buf, false)
780
log.Printf("net/trace: trace used after finish:\nFinished at:\n%s\nUsed at:\n%s", tr.finishStack, buf[:n])
786
If you are here because your program panicked in this code,
787
it is almost definitely the fault of code using this package,
788
and very unlikely to be the fault of this code.
790
The most likely scenario is that some code elsewhere is using
791
a trace.Trace after its Finish method is called.
792
You can temporarily set the DebugUseAfterFinish var
793
to help discover where that is; do not leave that var set,
794
since it makes this package much less efficient.
797
e := event{When: time.Now(), What: x, Recyclable: recyclable, Sensitive: sensitive}
799
e.Elapsed, e.NewDay = tr.delta(e.When)
800
if len(tr.events) < tr.maxEvents {
801
tr.events = append(tr.events, e)
803
// Discard the middle events.
804
di := int((tr.maxEvents - 1) / 2)
805
if d, ok := tr.events[di].What.(*discarded); ok {
808
// disc starts at two to count for the event it is replacing,
809
// plus the next one that we are about to drop.
811
if tr.recycler != nil && tr.events[di].Recyclable {
812
go tr.recycler(tr.events[di].What)
814
tr.events[di].What = &tr.disc
816
// The timestamp of the discarded meta-event should be
817
// the time of the last event it is representing.
818
tr.events[di].When = tr.events[di+1].When
820
if tr.recycler != nil && tr.events[di+1].Recyclable {
821
go tr.recycler(tr.events[di+1].What)
823
copy(tr.events[di+1:], tr.events[di+2:])
824
tr.events[tr.maxEvents-1] = e
829
func (tr *trace) LazyLog(x fmt.Stringer, sensitive bool) {
830
tr.addEvent(x, true, sensitive)
833
func (tr *trace) LazyPrintf(format string, a ...interface{}) {
834
tr.addEvent(&lazySprintf{format, a}, false, false)
837
func (tr *trace) SetError() {
843
func (tr *trace) SetRecycler(f func(interface{})) {
849
func (tr *trace) SetTraceInfo(traceID, spanID uint64) {
851
tr.traceID, tr.spanID = traceID, spanID
855
func (tr *trace) SetMaxEvents(m int) {
857
// Always keep at least three events: first, discarded count, last.
858
if len(tr.events) == 0 && m > 3 {
864
func (tr *trace) ref() {
865
atomic.AddInt32(&tr.refs, 1)
868
func (tr *trace) unref() {
869
if atomic.AddInt32(&tr.refs, -1) == 0 {
871
if tr.recycler != nil {
872
// freeTrace clears tr, so we hold tr.recycler and tr.events here.
873
go func(f func(interface{}), es []event) {
874
for _, e := range es {
879
}(tr.recycler, tr.events)
887
func (tr *trace) When() string {
888
return tr.Start.Format("2006/01/02 15:04:05.000000")
891
func (tr *trace) ElapsedTime() string {
898
t = time.Since(tr.Start)
900
return fmt.Sprintf("%.6f", t.Seconds())
903
func (tr *trace) Events() []event {
905
defer tr.mu.RUnlock()
909
var traceFreeList = make(chan *trace, 1000) // TODO(dsymonds): Use sync.Pool?
911
// newTrace returns a trace ready to use.
912
func newTrace() *trace {
914
case tr := <-traceFreeList:
921
// freeTrace adds tr to traceFreeList if there's room.
922
// This is non-blocking.
923
func freeTrace(tr *trace) {
924
if DebugUseAfterFinish {
925
return // never reuse
929
case traceFreeList <- tr:
934
func elapsed(d time.Duration) string {
935
b := []byte(fmt.Sprintf("%.6f", d.Seconds()))
937
// For subsecond durations, blank all zeros before decimal point,
938
// and all zeros between the decimal point and the first non-zero digit.
940
dot := bytes.IndexByte(b, '.')
941
for i := 0; i < dot; i++ {
944
for i := dot + 1; i < len(b); i++ {
956
var pageTmplCache *template.Template
957
var pageTmplOnce sync.Once
959
func pageTmpl() *template.Template {
960
pageTmplOnce.Do(func() {
961
pageTmplCache = template.Must(template.New("Page").Funcs(template.FuncMap{
963
"add": func(a, b int) int { return a + b },
970
{{template "Prolog" .}}
971
{{template "StatusTable" .}}
972
{{template "Epilog" .}}
977
<title>/debug/requests</title>
978
<style type="text/css">
980
font-family: sans-serif;
982
table#tr-status td.family {
985
table#tr-status td.active {
988
table#tr-status td.latency-first {
991
table#tr-status td.empty {
997
table#reqs tr.first {
998
{{if $.Expanded}}font-weight: bold;{{end}}
1001
font-family: monospace;
1003
table#reqs td.when {
1005
white-space: nowrap;
1007
table#reqs td.elapsed {
1021
<h1>/debug/requests</h1>
1022
{{end}} {{/* end of Prolog */}}
1024
{{define "StatusTable"}}
1025
<table id="tr-status">
1026
{{range $fam := .Families}}
1028
<td class="family">{{$fam}}</td>
1030
{{$n := index $.ActiveTraceCount $fam}}
1031
<td class="active {{if not $n}}empty{{end}}">
1032
{{if $n}}<a href="?fam={{$fam}}&b=-1{{if $.Expanded}}&exp=1{{end}}">{{end}}
1034
{{if $n}}</a>{{end}}
1037
{{$f := index $.CompletedTraces $fam}}
1038
{{range $i, $b := $f.Buckets}}
1039
{{$empty := $b.Empty}}
1040
<td {{if $empty}}class="empty"{{end}}>
1041
{{if not $empty}}<a href="?fam={{$fam}}&b={{$i}}{{if $.Expanded}}&exp=1{{end}}">{{end}}
1043
{{if not $empty}}</a>{{end}}
1047
{{$nb := len $f.Buckets}}
1048
<td class="latency-first">
1049
<a href="?fam={{$fam}}&b={{$nb}}">[minute]</a>
1052
<a href="?fam={{$fam}}&b={{add $nb 1}}">[hour]</a>
1055
<a href="?fam={{$fam}}&b={{add $nb 2}}">[total]</a>
1061
{{end}} {{/* end of StatusTable */}}
1066
<h3>Family: {{$.Family}}</h3>
1068
{{if or $.Expanded $.Traced}}
1069
<a href="?fam={{$.Family}}&b={{$.Bucket}}">[Normal/Summary]</a>
1074
{{if or (not $.Expanded) $.Traced}}
1075
<a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1">[Normal/Expanded]</a>
1081
{{if or $.Expanded (not $.Traced)}}
1082
<a href="?fam={{$.Family}}&b={{$.Bucket}}&rtraced=1">[Traced/Summary]</a>
1086
{{if or (not $.Expanded) (not $.Traced)}}
1087
<a href="?fam={{$.Family}}&b={{$.Bucket}}&exp=1&rtraced=1">[Traced/Expanded]</a>
1094
<p><em>Showing <b>{{len $.Traces}}</b> of <b>{{$.Total}}</b> traces.</em></p>
1099
{{if $.Active}}Active{{else}}Completed{{end}} Requests
1101
<tr><th>When</th><th>Elapsed (s)</th></tr>
1102
{{range $tr := $.Traces}}
1104
<td class="when">{{$tr.When}}</td>
1105
<td class="elapsed">{{$tr.ElapsedTime}}</td>
1106
<td>{{$tr.Title}}</td>
1107
{{/* TODO: include traceID/spanID */}}
1110
{{range $tr.Events}}
1112
<td class="when">{{.WhenString}}</td>
1113
<td class="elapsed">{{elapsed .Elapsed}}</td>
1114
<td>{{if or $.ShowSensitive (not .Sensitive)}}... {{.What}}{{else}}<em>[redacted]</em>{{end}}</td>
1120
{{end}} {{/* if $.Traces */}}
1123
<h4>Latency (µs) of {{$.Family}} over {{$.HistogramWindow}}</h4>
1125
{{end}} {{/* if $.Histogram */}}
1129
{{end}} {{/* end of Epilog */}}