prometheus-net

1
using System.Buffers;
2
using System.ComponentModel;
3
using System.Runtime.CompilerServices;
4
using System.Text.RegularExpressions;
5
using Microsoft.Extensions.ObjectPool;
6

7
namespace Prometheus;
8

9
/// <summary>
10
/// Base class for metrics, defining the basic informative API and the internal API.
11
/// </summary>
12
/// <remarks>
13
/// Many of the fields are lazy-initialized to ensure we only perform the memory allocation if and when we actually use them.
14
/// For some, it means rarely used members are never allocated at all (e.g. if you never inspect the set of label names, they are never allocated).
15
/// For others, it means they are allocated at first time of use (e.g. serialization-related fields are allocated when serializing the first time).
16
/// </remarks>
17
public abstract class Collector
18
{
19
    /// <summary>
20
    /// The metric name, e.g. http_requests_total.
21
    /// </summary>
22
    public string Name { get; }
23

24
    internal byte[] NameBytes => NonCapturingLazyInitializer.EnsureInitialized(ref _nameBytes, this, _assignNameBytesFunc)!;
25
    private byte[]? _nameBytes;
26
    private static readonly Action<Collector> _assignNameBytesFunc = AssignNameBytes;
27
    private static void AssignNameBytes(Collector instance) => instance._nameBytes = PrometheusConstants.ExportEncoding.GetBytes(instance.Name);
28

29
    /// <summary>
30
    /// The help text describing the metric for a human audience.
31
    /// </summary>
32
    public string Help { get; }
33

34
    internal byte[] HelpBytes => NonCapturingLazyInitializer.EnsureInitialized(ref _helpBytes, this, _assignHelpBytesFunc)!;
35
    private byte[]? _helpBytes;
36
    private static readonly Action<Collector> _assignHelpBytesFunc = AssignHelpBytes;
37
    private static void AssignHelpBytes(Collector instance) =>
38
        instance._helpBytes = string.IsNullOrWhiteSpace(instance.Help) ? [] : PrometheusConstants.ExportEncoding.GetBytes(instance.Help);
39

40
    /// <summary>
41
    /// Names of the instance-specific labels (name-value pairs) that apply to this metric.
42
    /// When the values are added to the names, you get a <see cref="ChildBase"/> instance.
43
    /// Does not include any static label names (from metric configuration, factory or registry).
44
    /// </summary>
45
    public string[] LabelNames => NonCapturingLazyInitializer.EnsureInitialized(ref _labelNames, this, _assignLabelNamesFunc)!;
46
    private string[]? _labelNames;
47
    private static readonly Action<Collector> _assignLabelNamesFunc = AssignLabelNames;
48
    private static void AssignLabelNames(Collector instance) => instance._labelNames = instance.InstanceLabelNames.ToArray();
49

50
    internal StringSequence InstanceLabelNames;
51
    internal StringSequence FlattenedLabelNames;
52

53
    /// <summary>
54
    /// All static labels obtained from any hierarchy level (either defined in metric configuration or in registry).
55
    /// These will be merged with the instance-specific labels to arrive at the final flattened label sequence for a specific child.
56
    /// </summary>
57
    internal LabelSequence StaticLabels;
58

59
    internal abstract MetricType Type { get; }
60

61
    internal byte[] TypeBytes { get; }
62

63
    internal abstract int ChildCount { get; }
64
    internal abstract int TimeseriesCount { get; }
65

66
    internal abstract ValueTask CollectAndSerializeAsync(IMetricsSerializer serializer, bool writeFamilyDeclaration, CancellationToken cancel);
67

68
    // Used by ChildBase.Remove()
69
    internal abstract void RemoveLabelled(LabelSequence instanceLabels);
70

71
    private const string ValidMetricNameExpression = "^[a-zA-Z_][a-zA-Z0-9_]*$";
72
    private const string ValidLabelNameExpression = "^[a-zA-Z_][a-zA-Z0-9_]*$";
73
    private const string ReservedLabelNameExpression = "^__.*$";
74

75
    private static readonly Regex MetricNameRegex = new(ValidMetricNameExpression, RegexOptions.Compiled);
76
    private static readonly Regex LabelNameRegex = new(ValidLabelNameExpression, RegexOptions.Compiled);
77
    private static readonly Regex ReservedLabelRegex = new(ReservedLabelNameExpression, RegexOptions.Compiled);
78

79
    internal Collector(string name, string help, StringSequence instanceLabelNames, LabelSequence staticLabels)
80
    {
81
        if (!MetricNameRegex.IsMatch(name))
82
            throw new ArgumentException($"Metric name '{name}' does not match regex '{ValidMetricNameExpression}'.");
83

84
        Name = name;
85
        TypeBytes = TextSerializer.MetricTypeToBytes[Type];
86
        Help = help;
87
        InstanceLabelNames = instanceLabelNames;
88
        StaticLabels = staticLabels;
89

90
        FlattenedLabelNames = instanceLabelNames.Concat(staticLabels.Names);
91

92
        // Used to check uniqueness of label names, to catch any label layering mistakes early.
93
        var uniqueLabelNames = LabelValidationHashSetPool.Get();
94

95
        try
96
        {
97
            foreach (var labelName in FlattenedLabelNames)
98
            {
99
                if (labelName == null)
100
                    throw new ArgumentException("One of the label names was null.");
101

102
                ValidateLabelName(labelName);
103
                uniqueLabelNames.Add(labelName);
104
            }
105

106
            // Here we check for label name collision, ensuring that the same label name is not defined twice on any label inheritance level.
107
            if (uniqueLabelNames.Count != FlattenedLabelNames.Length)
108
                throw new InvalidOperationException("The set of label names includes duplicates: " + string.Join(", ", FlattenedLabelNames.ToArray()));
109
        }
110
        finally
111
        {
112
            LabelValidationHashSetPool.Return(uniqueLabelNames);
113
        }
114
    }
115

116
    private static readonly ObjectPool<HashSet<string>> LabelValidationHashSetPool = ObjectPool.Create(new LabelValidationHashSetPoolPolicy());
117

118
    private sealed class LabelValidationHashSetPoolPolicy : PooledObjectPolicy<HashSet<string>>
119
    {
120
        // If something should explode the size, we do not return it to the pool.
121
        // This should be more than generous even for the most verbosely labeled scenarios.
122
        private const int PooledHashSetMaxSize = 50;
123

124
#if NET
125
        public override HashSet<string> Create() => new(PooledHashSetMaxSize, StringComparer.Ordinal);
126
#else
127
        public override HashSet<string> Create() => new(StringComparer.Ordinal);
128
#endif
129

130
        public override bool Return(HashSet<string> obj)
131
        {
132
            if (obj.Count > PooledHashSetMaxSize)
133
                return false;
134

135
            obj.Clear();
136
            return true;
137
        }
138
    }
139

140
    internal static void ValidateLabelName(string labelName)
141
    {
142
        if (!LabelNameRegex.IsMatch(labelName))
143
            throw new ArgumentException($"Label name '{labelName}' does not match regex '{ValidLabelNameExpression}'.");
144

145
        if (ReservedLabelRegex.IsMatch(labelName))
146
            throw new ArgumentException($"Label name '{labelName}' is not valid - labels starting with double underscore are reserved!");
147
    }
148

149
    public override string ToString()
150
    {
151
        // Just for debugging.
152
        return $"{Name}{{{FlattenedLabelNames}}}";
153
    }
154
}
155

156
/// <summary>
157
/// Base class for metrics collectors, providing common labeled child management functionality.
158
/// </summary>
159
public abstract class Collector<TChild> : Collector, ICollector<TChild>
160
    where TChild : ChildBase
161
{
162
    // Keyed by the instance labels (not by flattened labels!).
163
    private readonly Dictionary<LabelSequence, TChild> _children = [];
164
    private readonly ReaderWriterLockSlim _childrenLock = new();
165

166
    // Lazy-initialized since not every collector will use a child with no labels.
167
    // Lazy instance will be replaced if the unlabelled timeseries is removed.
168
    private TChild? _lazyUnlabelled;
169

170
    /// <summary>
171
    /// Gets the child instance that has no labels.
172
    /// </summary>
173
    protected internal TChild Unlabelled => LazyInitializer.EnsureInitialized(ref _lazyUnlabelled, _createdUnlabelledFunc)!;
174

175
    private TChild CreateUnlabelled() => GetOrAddLabelled(LabelSequence.Empty);
176
    private readonly Func<TChild> _createdUnlabelledFunc;
177

178
    // We need it for the ICollector interface but using this is rarely relevant in client code, so keep it obscured.
179
    TChild ICollector<TChild>.Unlabelled => Unlabelled;
180

181

182
    // Old naming, deprecated for a silly reason: by default if you start typing .La... and trigger Intellisense
183
    // it will often for whatever reason focus on LabelNames instead of Labels, leading to tiny but persistent frustration.
184
    // Having WithLabels() instead eliminates the other candidate and allows for a frustration-free typing experience.
185
    // Discourage this method as it can create confusion. But it works fine, so no reason to mark it obsolete, really.
186
    [EditorBrowsable(EditorBrowsableState.Never)]
187
    public TChild Labels(params string[] labelValues) => WithLabels(labelValues);
188

189
    public TChild WithLabels(params string[] labelValues)
190
    {
191
        if (labelValues == null)
192
            throw new ArgumentNullException(nameof(labelValues));
193

194
        return WithLabels(labelValues.AsMemory());
195
    }
196

197
    public TChild WithLabels(ReadOnlyMemory<string> labelValues)
198
    {
199
        var labels = LabelSequence.From(InstanceLabelNames, StringSequence.From(labelValues));
200
        return GetOrAddLabelled(labels);
201
    }
202

203
    public TChild WithLabels(ReadOnlySpan<string> labelValues)
204
    {
205
        // We take ReadOnlySpan as a signal that the caller believes we may be able to perform the operation allocation-free because
206
        // the label values are probably already known and a metric instance registered. There is no a guarantee, just a high probability.
207
        // The implementation avoids allocating a long-lived string[] for the label values. We only allocate if we create a new instance.
208

209
        // We still need to process the label values as a reference type, so we transform the Span into a Memory using a pooled buffer.
210
        var buffer = ArrayPool<string>.Shared.Rent(labelValues.Length);
211

212
        try
213
        {
214
            labelValues.CopyTo(buffer);
215

216
            var temporaryLabels = LabelSequence.From(InstanceLabelNames, StringSequence.From(buffer.AsMemory(0, labelValues.Length)));
217

218
            if (TryGetLabelled(temporaryLabels, out var existing))
219
                return existing!;
220
        }
221
        finally
222
        {
223
            ArrayPool<string>.Shared.Return(buffer);
224
        }
225

226
        // If we got this far, we did not succeed in finding an existing instance. We need to allocate a long-lived string[] for the label values.
227
        var labels = LabelSequence.From(InstanceLabelNames, StringSequence.From(labelValues.ToArray()));
228
        return CreateLabelled(labels);
229
    }
230

231
    public void RemoveLabelled(params string[] labelValues)
232
    {
233
        if (labelValues == null)
234
            throw new ArgumentNullException(nameof(labelValues));
235

236
        var labels = LabelSequence.From(InstanceLabelNames, StringSequence.From(labelValues));
237
        RemoveLabelled(labels);
238
    }
239

240
    internal override void RemoveLabelled(LabelSequence labels)
241
    {
242
        _childrenLock.EnterWriteLock();
243

244
        try
245
        {
246
            _children.Remove(labels);
247

248
            if (labels.Length == 0)
249
            {
250
                // If we remove the unlabeled instance (technically legitimate, if the caller really desires to do so) then
251
                // we need to also ensure that the special-casing used for it gets properly wired up the next time.
252
                Volatile.Write(ref _lazyUnlabelled, null);
253
            }
254
        }
255
        finally
256
        {
257
            _childrenLock.ExitWriteLock();
258
        }
259
    }
260

261
    internal override int ChildCount
262
    {
263
        get
264
        {
265
            _childrenLock.EnterReadLock();
266

267
            try
268
            {
269
                return _children.Count;
270
            }
271
            finally
272
            {
273
                _childrenLock.ExitReadLock();
274
            }
275
        }
276
    }
277

278
    /// <summary>
279
    /// Gets the instance-specific label values of all labelled instances of the collector.
280
    /// Values of any inherited static labels are not returned in the result.
281
    /// 
282
    /// Note that during concurrent operation, the set of values returned here
283
    /// may diverge from the latest set of values used by the collector.
284
    /// </summary>
285
    public IEnumerable<string[]> GetAllLabelValues()
286
    {
287
        // We are yielding here so make a defensive copy so we do not hold locks for a long time.
288
        // We reuse this buffer, so it should be relatively harmless in the long run.
289
        LabelSequence[] buffer;
290

291
        _childrenLock.EnterReadLock();
292

293
        var childCount = _children.Count;
294
        buffer = ArrayPool<LabelSequence>.Shared.Rent(childCount);
295

296
        try
297
        {
298
            try
299
            {
300
                _children.Keys.CopyTo(buffer, 0);
301
            }
302
            finally
303
            {
304
                _childrenLock.ExitReadLock();
305
            }
306

307
            for (var i = 0; i < childCount; i++)
308
            {
309
                var labels = buffer[i];
310

311
                if (labels.Length == 0)
312
                    continue; // We do not return the "unlabelled" label set.
313

314
                // Defensive copy.
315
                yield return labels.Values.ToArray();
316
            }
317
        }
318
        finally
319
        {
320
            ArrayPool<LabelSequence>.Shared.Return(buffer);
321
        }
322
    }
323

324
    private TChild GetOrAddLabelled(LabelSequence instanceLabels)
325
    {
326
        // NOTE: We do not try to find a metric instance with the same set of label names but in a DIFFERENT order.
327
        // Order of labels matters in data creation, although does not matter when the exported data set is imported later.
328
        // If we somehow end up registering the same metric with the same label names in different order, we will publish it twice, in two orders...
329
        // That is not ideal but also not that big of a deal to justify a lookup every time a metric instance is registered.
330

331
        // First try to find an existing instance. This is the fast path, if we are re-looking-up an existing one.
332
        if (TryGetLabelled(instanceLabels, out var existing))
333
            return existing!;
334

335
        // If no existing one found, grab the write lock and create a new one if needed.
336
        return CreateLabelled(instanceLabels);
337
    }
338

339
    private bool TryGetLabelled(LabelSequence instanceLabels, out TChild? child)
340
    {
341
        _childrenLock.EnterReadLock();
342

343
        try
344
        {
345
            if (_children.TryGetValue(instanceLabels, out var existing))
346
            {
347
                child = existing;
348
                return true;
349
            }
350

351
            child = null;
352
            return false;
353
        }
354
        finally
355
        {
356
            _childrenLock.ExitReadLock();
357
        }
358
    }
359

360
    private TChild CreateLabelled(LabelSequence instanceLabels)
361
    {
362
        var newChild = _createdLabelledChildFunc(instanceLabels);
363

364
        _childrenLock.EnterWriteLock();
365

366
        try
367
        {
368
#if NET
369
            // It could be that someone beats us to it! Probably not, though.
370
            if (_children.TryAdd(instanceLabels, newChild))
371
                return newChild;
372

373
            return _children[instanceLabels];
374
#else
375
            // On .NET Fx we need to do the pessimistic case first because there is no TryAdd().
376
            if (_children.TryGetValue(instanceLabels, out var existing))
377
                return existing;
378

379
            _children.Add(instanceLabels, newChild);
380
            return newChild;
381
#endif
382
        }
383
        finally
384
        {
385
            _childrenLock.ExitWriteLock();
386
        }
387
    }
388

389
    private TChild CreateLabelledChild(LabelSequence instanceLabels)
390
    {
391
        // Order of labels is 1) instance labels; 2) static labels.
392
        var flattenedLabels = instanceLabels.Concat(StaticLabels);
393

394
        return NewChild(instanceLabels, flattenedLabels, publish: !_suppressInitialValue, _exemplarBehavior);
395
    }
396

397
    // Cache the delegate to avoid allocating a new one every time in GetOrAddLabelled.
398
    private readonly Func<LabelSequence, TChild> _createdLabelledChildFunc;
399

400
    /// <summary>
401
    /// For tests that want to see what instance-level label values were used when metrics were created.
402
    /// This is for testing only, so does not respect locks - do not use this in concurrent context.
403
    /// </summary>
404
    internal LabelSequence[] GetAllInstanceLabelsUnsafe() => _children.Keys.ToArray();
405

406
    internal Collector(string name, string help, StringSequence instanceLabelNames, LabelSequence staticLabels, bool suppressInitialValue, ExemplarBehavior exemplarBehavior)
407
        : base(name, help, instanceLabelNames, staticLabels)
408
    {
409
        _createdUnlabelledFunc = CreateUnlabelled;
410
        _createdLabelledChildFunc = CreateLabelledChild;
411

412
        _suppressInitialValue = suppressInitialValue;
413
        _exemplarBehavior = exemplarBehavior;
414
    }
415

416
    /// <summary>
417
    /// Creates a new instance of the child collector type.
418
    /// </summary>
419
    private protected abstract TChild NewChild(LabelSequence instanceLabels, LabelSequence flattenedLabels, bool publish, ExemplarBehavior exemplarBehavior);
420

421
#if NET
422
    [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))]
423
#endif
424
    internal override async ValueTask CollectAndSerializeAsync(IMetricsSerializer serializer, bool writeFamilyDeclaration, CancellationToken cancel)
425
    {
426
        EnsureUnlabelledMetricCreatedIfNoLabels();
427

428
        // There may be multiple Collectors emitting data for the same family. Only the first will write out the family declaration.
429
        if (writeFamilyDeclaration)
430
            await serializer.WriteFamilyDeclarationAsync(Name, NameBytes, HelpBytes, Type, TypeBytes, cancel);
431

432
        // This could potentially take nontrivial time, as we are serializing to a stream (potentially, a network stream).
433
        // Therefore we operate on a defensive copy in a reused buffer.
434
        TChild[] children;
435

436
        _childrenLock.EnterReadLock();
437

438
        var childCount = _children.Count;
439
        children = ArrayPool<TChild>.Shared.Rent(childCount);
440

441
        try
442
        {
443
            try
444
            {
445
                _children.Values.CopyTo(children, 0);
446
            }
447
            finally
448
            {
449
                _childrenLock.ExitReadLock();
450
            }
451

452
            for (var i = 0; i < childCount; i++)
453
            {
454
                var child = children[i];
455
                await child.CollectAndSerializeAsync(serializer, cancel);
456
            }
457
        }
458
        finally
459
        {
460
            ArrayPool<TChild>.Shared.Return(children, clearArray: true);
461
        }
462
    }
463

464
    private readonly bool _suppressInitialValue;
465

466
    private void EnsureUnlabelledMetricCreatedIfNoLabels()
467
    {
468
        // We want metrics to exist even with 0 values if they are supposed to be used without labels.
469
        // Labelled metrics are created when label values are assigned. However, as unlabelled metrics are lazy-created
470
        // (they are optional if labels are used) we might lose them for cases where they really are desired.
471

472
        // If there are no label names then clearly this metric is supposed to be used unlabelled, so create it.
473
        // Otherwise, we allow unlabelled metrics to be used if the user explicitly does it but omit them by default.
474
        if (InstanceLabelNames.Length == 0)
475
            LazyInitializer.EnsureInitialized(ref _lazyUnlabelled, _createdUnlabelledFunc);
476
    }
477

478
    private readonly ExemplarBehavior _exemplarBehavior;
479
}