prometheus-net

1
using System.Collections.Concurrent;
2
using System.Diagnostics;
3
using System.Diagnostics.Tracing;
4
using System.Globalization;
5

6
namespace Prometheus;
7

8
/// <summary>
9
/// Monitors .NET EventCounters and exposes them as Prometheus metrics.
10
/// </summary>
11
/// <remarks>
12
/// All observed .NET event counters are transformed into Prometheus metrics with translated names.
13
/// </remarks>
14
public sealed class EventCounterAdapter : IDisposable
15
{
16
    public static IDisposable StartListening() => StartListening(EventCounterAdapterOptions.Default);
17

18
    public static IDisposable StartListening(EventCounterAdapterOptions options)
19
    {
20
        // If we are re-registering an adapter with the default options, just pretend and move on.
21
        // The purpose of this code is to avoid double-counting metrics if the adapter is registered twice with the default options.
22
        // This could happen because in 7.0.0 we added automatic registration of the adapters on startup, but the user might still
23
        // have a manual registration active from 6.0.0 days. We do this small thing here to make upgrading less hassle.
24
        if (options == EventCounterAdapterOptions.Default)
25
        {
26
            if (options.Registry.PreventEventCounterAdapterRegistrationWithDefaultOptions)
27
                return new NoopDisposable();
28

29
            options.Registry.PreventEventCounterAdapterRegistrationWithDefaultOptions = true;
30
        }
31

32
        return new EventCounterAdapter(options);
33
    }
34

35
    private EventCounterAdapter(EventCounterAdapterOptions options)
36
    {
37
        _options = options;
38
        _metricFactory = _options.MetricFactory ?? Metrics.WithCustomRegistry(_options.Registry);
39

40
        _eventSourcesConnected = _metricFactory.CreateGauge("prometheus_net_eventcounteradapter_sources_connected_total", "Number of event sources that are currently connected to the adapter.");
41

42
        EventCounterAdapterMemoryWarden.EnsureStarted();
43

44
        _listener = new Listener(ShouldUseEventSource, ConfigureEventSource, options.UpdateInterval, OnEventWritten);
45
    }
46

47
    public void Dispose()
48
    {
49
        // Disposal means we stop listening but we do not remove any published data just to keep things simple.
50
        _listener.Dispose();
51
    }
52

53
    private readonly EventCounterAdapterOptions _options;
54
    private readonly IMetricFactory _metricFactory;
55

56
    private readonly Listener _listener;
57

58
    // We never decrease it in the current implementation but perhaps might in a future implementation, so might as well make it a gauge.
59
    private readonly Gauge _eventSourcesConnected;
60

61
    private bool ShouldUseEventSource(EventSource source)
62
    {
63
        bool connect = _options.EventSourceFilterPredicate(source.Name);
64

65
        if (connect)
66
            _eventSourcesConnected.Inc();
67

68
        return connect;
69
    }
70

71
    private EventCounterAdapterEventSourceSettings ConfigureEventSource(EventSource source)
72
    {
73
        return _options.EventSourceSettingsProvider(source.Name);
74
    }
75

76
    private const string RateSuffix = "_rate";
77

78
    private void OnEventWritten(EventWrittenEventArgs args)
79
    {
80
        // This deserialization here is pretty gnarly.
81
        // We just skip anything that makes no sense.
82

83
        try
84
        {
85
            if (args.EventName != "EventCounters")
86
                return; // Do not know what it is and do not care.
87

88
            if (args.Payload == null)
89
                return; // What? Whatever.
90

91
            var eventSourceName = args.EventSource.Name;
92

93
            foreach (var item in args.Payload)
94
            {
95
                if (item is not IDictionary<string, object> e)
96
                    continue;
97

98
                if (!e.TryGetValue("Name", out var nameWrapper))
99
                    continue;
100

101
                var name = nameWrapper as string;
102

103
                if (name == null)
104
                    continue; // What? Whatever.
105

106
                if (!e.TryGetValue("DisplayName", out var displayNameWrapper))
107
                    continue;
108

109
                var displayName = displayNameWrapper as string ?? "";
110

111
                // If there is a DisplayUnits, prefix it to the help text.
112
                if (e.TryGetValue("DisplayUnits", out var displayUnitsWrapper) && !string.IsNullOrWhiteSpace(displayUnitsWrapper as string))
113
                    displayName = $"({(string)displayUnitsWrapper}) {displayName}";
114

115
                var mergedName = $"{eventSourceName}_{name}";
116

117
                var prometheusName = _counterPrometheusName.GetOrAdd(mergedName, PrometheusNameHelpers.TranslateNameToPrometheusName);
118

119
                // The event counter can either be
120
                // 1) an aggregating counter (in which case we use the mean); or
121
                // 2) an incrementing counter (in which case we use the delta).
122

123
                if (e.TryGetValue("Increment", out var increment))
124
                {
125
                    // Looks like an incrementing counter.
126

127
                    var value = increment as double?;
128

129
                    if (value == null)
130
                        continue; // What? Whatever.
131

132
                    // If the underlying metric is exposing a rate then this can result in some strange terminology like "rate_total".
133
                    // We will remove the "rate" from the name to be more understandable - you'll get the rate when you apply the Prometheus rate() function, the raw value is not the rate.
134
                    if (prometheusName.EndsWith(RateSuffix))
135
                        prometheusName = prometheusName.Remove(prometheusName.Length - RateSuffix.Length);
136

137
                    _metricFactory.CreateCounter(prometheusName + "_total", displayName).Inc(value.Value);
138
                }
139
                else if (e.TryGetValue("Mean", out var mean))
140
                {
141
                    // Looks like an aggregating counter.
142

143
                    var value = mean as double?;
144

145
                    if (value == null)
146
                        continue; // What? Whatever.
147

148
                    _metricFactory.CreateGauge(prometheusName, displayName).Set(value.Value);
149
                }
150
            }
151
        }
152
        catch (Exception ex)
153
        {
154
            // We do not want to throw any exceptions if we fail to handle this event because who knows what it messes up upstream.
155
            Trace.WriteLine($"Failed to parse EventCounter event: {ex.Message}");
156
        }
157
    }
158

159
    // Source+Name -> Name
160
    private readonly ConcurrentDictionary<string, string> _counterPrometheusName = new();
161

162
    private sealed class Listener : EventListener
163
    {
164
        public Listener(
165
            Func<EventSource, bool> shouldUseEventSource,
166
            Func<EventSource, EventCounterAdapterEventSourceSettings> configureEventSosurce,
167
            TimeSpan updateInterval,
168
            Action<EventWrittenEventArgs> onEventWritten)
169
        {
170
            _shouldUseEventSource = shouldUseEventSource;
171
            _configureEventSosurce = configureEventSosurce;
172
            _updateInterval = updateInterval;
173
            _onEventWritten = onEventWritten;
174

175
            foreach (var eventSource in _preRegisteredEventSources)
176
                OnEventSourceCreated(eventSource);
177

178
            _preRegisteredEventSources.Clear();
179
        }
180

181
        private readonly List<EventSource> _preRegisteredEventSources = new List<EventSource>();
182

183
        private readonly Func<EventSource, bool> _shouldUseEventSource;
184
        private readonly Func<EventSource, EventCounterAdapterEventSourceSettings> _configureEventSosurce;
185
        private readonly TimeSpan _updateInterval;
186
        private readonly Action<EventWrittenEventArgs> _onEventWritten;
187

188
        protected override void OnEventSourceCreated(EventSource eventSource)
189
        {
190
            if (_shouldUseEventSource == null)
191
            {
192
                // The way this EventListener thing works is rather strange. Immediately in the base class constructor, before we
193
                // have even had time to wire up our subclass, it starts calling OnEventSourceCreated for all already-existing event sources...
194
                // We just buffer those calls because CALM DOWN SIR!
195
                _preRegisteredEventSources.Add(eventSource);
196
                return;
197
            }
198

199
            if (!_shouldUseEventSource(eventSource))
200
                return;
201

202
            try
203
            {
204
                var options = _configureEventSosurce(eventSource);
205

206
                EnableEvents(eventSource, options.MinimumLevel, options.MatchKeywords, new Dictionary<string, string?>()
207
                {
208
                    ["EventCounterIntervalSec"] = ((int)Math.Max(1, _updateInterval.TotalSeconds)).ToString(CultureInfo.InvariantCulture),
209
                });
210
            }
211
            catch (Exception ex)
212
            {
213
                // Eat exceptions here to ensure no harm comes of failed enabling.
214
                // The EventCounter infrastructure has proven quite buggy and while it is not certain that this may throw, let's be paranoid.
215
                Trace.WriteLine($"Failed to enable EventCounter listening for {eventSource.Name}: {ex.Message}");
216
            }
217
        }
218

219
        protected override void OnEventWritten(EventWrittenEventArgs eventData)
220
        {
221
            _onEventWritten(eventData);
222
        }
223
    }
224

225
    /// <summary>
226
    /// By default we enable event sources that start with any of these strings. This is a manually curated list to try enable some useful ones
227
    /// without just enabling everything under the sky (because .NET has no way to say "enable only the event counters", you have to enable all diagnostic events).
228
    /// </summary>
229
    private static readonly IReadOnlyList<string> DefaultEventSourcePrefixes = new[]
230
    {
231
        "System.Runtime",
232
        "Microsoft-AspNetCore",
233
        "Microsoft.AspNetCore",
234
        "System.Net"
235
    };
236

237
    public static readonly Func<string, bool> DefaultEventSourceFilterPredicate = name => DefaultEventSourcePrefixes.Any(x => name.StartsWith(x, StringComparison.Ordinal));
238
}
239