weaviate

1
{
2
  "annotations": {
3
    "list": [
4
      {
5
        "builtIn": 1,
6
        "datasource": "-- Grafana --",
7
        "enable": true,
8
        "hide": true,
9
        "iconColor": "rgba(0, 211, 255, 1)",
10
        "name": "Annotations & Alerts",
11
        "target": {
12
          "limit": 100,
13
          "matchAny": false,
14
          "tags": [],
15
          "type": "dashboard"
16
        },
17
        "type": "dashboard"
18
      }
19
    ]
20
  },
21
  "editable": true,
22
  "fiscalYearStartMonth": 0,
23
  "graphTooltip": 0,
24
  "links": [],
25
  "liveNow": false,
26
  "panels": [
27
    {
28
      "datasource": {
29
        "type": "prometheus",
30
        "uid": "Prometheus"
31
      },
32
      "fieldConfig": {
33
        "defaults": {
34
          "color": {
35
            "mode": "palette-classic"
36
          },
37
          "custom": {
38
            "axisLabel": "",
39
            "axisPlacement": "auto",
40
            "barAlignment": 0,
41
            "drawStyle": "line",
42
            "fillOpacity": 0,
43
            "gradientMode": "none",
44
            "hideFrom": {
45
              "legend": false,
46
              "tooltip": false,
47
              "viz": false
48
            },
49
            "lineInterpolation": "linear",
50
            "lineWidth": 1,
51
            "pointSize": 5,
52
            "scaleDistribution": {
53
              "type": "linear"
54
            },
55
            "showPoints": "auto",
56
            "spanNulls": false,
57
            "stacking": {
58
              "group": "A",
59
              "mode": "none"
60
            },
61
            "thresholdsStyle": {
62
              "mode": "off"
63
            }
64
          },
65
          "mappings": [],
66
          "thresholds": {
67
            "mode": "absolute",
68
            "steps": [
69
              {
70
                "color": "green",
71
                "value": null
72
              },
73
              {
74
                "color": "red",
75
                "value": 80
76
              }
77
            ]
78
          },
79
          "unit": "ms"
80
        },
81
        "overrides": []
82
      },
83
      "gridPos": {
84
        "h": 9,
85
        "w": 12,
86
        "x": 0,
87
        "y": 0
88
      },
89
      "id": 2,
90
      "interval": "3",
91
      "options": {
92
        "legend": {
93
          "calcs": [],
94
          "displayMode": "list",
95
          "placement": "bottom"
96
        },
97
        "tooltip": {
98
          "mode": "single"
99
        }
100
      },
101
      "targets": [
102
        {
103
          "datasource": {
104
            "type": "prometheus",
105
            "uid": "Prometheus"
106
          },
107
          "exemplar": true,
108
          "expr": "rate(batch_durations_ms_sum{class_name=\"n/a\"}[$__interval])/rate(batch_durations_ms_count{class_name=\"n/a\"}[$__interval])",
109
          "interval": "",
110
          "legendFormat": "{{operation}}",
111
          "refId": "A"
112
        }
113
      ],
114
      "title": "Batch Objects Latency (Components & Totals)",
115
      "type": "timeseries"
116
    },
117
    {
118
      "datasource": {
119
        "type": "prometheus",
120
        "uid": "Prometheus"
121
      },
122
      "description": "",
123
      "fieldConfig": {
124
        "defaults": {
125
          "color": {
126
            "mode": "palette-classic"
127
          },
128
          "custom": {
129
            "axisLabel": "",
130
            "axisPlacement": "auto",
131
            "barAlignment": 0,
132
            "drawStyle": "line",
133
            "fillOpacity": 0,
134
            "gradientMode": "none",
135
            "hideFrom": {
136
              "legend": false,
137
              "tooltip": false,
138
              "viz": false
139
            },
140
            "lineInterpolation": "linear",
141
            "lineWidth": 1,
142
            "pointSize": 5,
143
            "scaleDistribution": {
144
              "type": "linear"
145
            },
146
            "showPoints": "auto",
147
            "spanNulls": false,
148
            "stacking": {
149
              "group": "A",
150
              "mode": "none"
151
            },
152
            "thresholdsStyle": {
153
              "mode": "off"
154
            }
155
          },
156
          "mappings": [],
157
          "thresholds": {
158
            "mode": "absolute",
159
            "steps": [
160
              {
161
                "color": "green",
162
                "value": null
163
              },
164
              {
165
                "color": "red",
166
                "value": 80
167
              }
168
            ]
169
          }
170
        },
171
        "overrides": []
172
      },
173
      "gridPos": {
174
        "h": 9,
175
        "w": 9,
176
        "x": 12,
177
        "y": 0
178
      },
179
      "id": 9,
180
      "interval": "1",
181
      "options": {
182
        "legend": {
183
          "calcs": [],
184
          "displayMode": "list",
185
          "placement": "bottom"
186
        },
187
        "tooltip": {
188
          "mode": "single"
189
        }
190
      },
191
      "targets": [
192
        {
193
          "datasource": {
194
            "type": "prometheus",
195
            "uid": "Prometheus"
196
          },
197
          "exemplar": true,
198
          "expr": "vector_index_tombstones{}",
199
          "instant": false,
200
          "interval": "",
201
          "intervalFactor": 1,
202
          "legendFormat": "{{class_name}} - {{shard_name}}",
203
          "refId": "A"
204
        }
205
      ],
206
      "title": "Active Tombstones in HNSW index",
207
      "type": "timeseries"
208
    },
209
    {
210
      "datasource": {
211
        "type": "prometheus",
212
        "uid": "Prometheus"
213
      },
214
      "fieldConfig": {
215
        "defaults": {
216
          "color": {
217
            "mode": "thresholds"
218
          },
219
          "mappings": [],
220
          "thresholds": {
221
            "mode": "absolute",
222
            "steps": [
223
              {
224
                "color": "green",
225
                "value": null
226
              },
227
              {
228
                "color": "red",
229
                "value": 100000
230
              }
231
            ]
232
          }
233
        },
234
        "overrides": []
235
      },
236
      "gridPos": {
237
        "h": 9,
238
        "w": 3,
239
        "x": 21,
240
        "y": 0
241
      },
242
      "id": 11,
243
      "options": {
244
        "orientation": "auto",
245
        "reduceOptions": {
246
          "calcs": [
247
            "lastNotNull"
248
          ],
249
          "fields": "",
250
          "values": false
251
        },
252
        "showThresholdLabels": false,
253
        "showThresholdMarkers": true
254
      },
255
      "pluginVersion": "8.3.3",
256
      "targets": [
257
        {
258
          "datasource": {
259
            "type": "prometheus",
260
            "uid": "Prometheus"
261
          },
262
          "exemplar": true,
263
          "expr": "sum(vector_index_tombstones{})",
264
          "interval": "",
265
          "legendFormat": "",
266
          "refId": "A"
267
        }
268
      ],
269
      "title": "Currently active HNSW tombstones",
270
      "type": "gauge"
271
    },
272
    {
273
      "datasource": {
274
        "type": "prometheus",
275
        "uid": "Prometheus"
276
      },
277
      "fieldConfig": {
278
        "defaults": {
279
          "color": {
280
            "mode": "palette-classic"
281
          },
282
          "custom": {
283
            "axisLabel": "",
284
            "axisPlacement": "auto",
285
            "barAlignment": 0,
286
            "drawStyle": "line",
287
            "fillOpacity": 0,
288
            "gradientMode": "none",
289
            "hideFrom": {
290
              "legend": false,
291
              "tooltip": false,
292
              "viz": false
293
            },
294
            "lineInterpolation": "linear",
295
            "lineWidth": 1,
296
            "pointSize": 5,
297
            "scaleDistribution": {
298
              "type": "linear"
299
            },
300
            "showPoints": "auto",
301
            "spanNulls": false,
302
            "stacking": {
303
              "group": "A",
304
              "mode": "none"
305
            },
306
            "thresholdsStyle": {
307
              "mode": "off"
308
            }
309
          },
310
          "mappings": [],
311
          "thresholds": {
312
            "mode": "absolute",
313
            "steps": [
314
              {
315
                "color": "green",
316
                "value": null
317
              },
318
              {
319
                "color": "red",
320
                "value": 80
321
              }
322
            ]
323
          },
324
          "unit": "ms"
325
        },
326
        "overrides": []
327
      },
328
      "gridPos": {
329
        "h": 9,
330
        "w": 12,
331
        "x": 0,
332
        "y": 9
333
      },
334
      "id": 5,
335
      "interval": "3",
336
      "options": {
337
        "legend": {
338
          "calcs": [],
339
          "displayMode": "table",
340
          "placement": "bottom"
341
        },
342
        "tooltip": {
343
          "mode": "single"
344
        }
345
      },
346
      "targets": [
347
        {
348
          "datasource": {
349
            "type": "prometheus",
350
            "uid": "Prometheus"
351
          },
352
          "exemplar": true,
353
          "expr": "rate(batch_durations_ms_sum{operation=\"object_storage\"}[$__interval])/rate(batch_durations_ms_count{operation=\"object_storage\"}[$__interval])",
354
          "interval": "",
355
          "legendFormat": "Object Storage ({{class_name}} - {{shard_name}})",
356
          "refId": "A"
357
        },
358
        {
359
          "datasource": {
360
            "type": "prometheus",
361
            "uid": "Prometheus"
362
          },
363
          "exemplar": true,
364
          "expr": "rate(batch_durations_ms_sum{operation=\"vector_storage\"}[$__interval])/rate(batch_durations_ms_count{operation=\"vector_storage\"}[$__interval])",
365
          "hide": false,
366
          "interval": "",
367
          "legendFormat": "Vector Storage ({{class_name}} - {{shard_name}})",
368
          "refId": "B"
369
        }
370
      ],
371
      "title": "Persistence Tasks (Object, Vector)",
372
      "type": "timeseries"
373
    },
374
    {
375
      "datasource": {
376
        "type": "prometheus",
377
        "uid": "Prometheus"
378
      },
379
      "fieldConfig": {
380
        "defaults": {
381
          "color": {
382
            "mode": "palette-classic"
383
          },
384
          "custom": {
385
            "axisLabel": "",
386
            "axisPlacement": "auto",
387
            "barAlignment": 0,
388
            "drawStyle": "line",
389
            "fillOpacity": 0,
390
            "gradientMode": "none",
391
            "hideFrom": {
392
              "legend": false,
393
              "tooltip": false,
394
              "viz": false
395
            },
396
            "lineInterpolation": "linear",
397
            "lineWidth": 1,
398
            "pointSize": 5,
399
            "scaleDistribution": {
400
              "type": "linear"
401
            },
402
            "showPoints": "auto",
403
            "spanNulls": false,
404
            "stacking": {
405
              "group": "A",
406
              "mode": "none"
407
            },
408
            "thresholdsStyle": {
409
              "mode": "off"
410
            }
411
          },
412
          "mappings": [],
413
          "thresholds": {
414
            "mode": "absolute",
415
            "steps": [
416
              {
417
                "color": "green",
418
                "value": null
419
              },
420
              {
421
                "color": "red",
422
                "value": 80
423
              }
424
            ]
425
          }
426
        },
427
        "overrides": []
428
      },
429
      "gridPos": {
430
        "h": 8,
431
        "w": 6,
432
        "x": 12,
433
        "y": 9
434
      },
435
      "id": 13,
436
      "options": {
437
        "legend": {
438
          "calcs": [],
439
          "displayMode": "list",
440
          "placement": "bottom"
441
        },
442
        "tooltip": {
443
          "mode": "single"
444
        }
445
      },
446
      "targets": [
447
        {
448
          "datasource": {
449
            "type": "prometheus",
450
            "uid": "Prometheus"
451
          },
452
          "exemplar": true,
453
          "expr": "vector_index_tombstone_cleanup_threads{}",
454
          "interval": "",
455
          "legendFormat": "{{class_name}} - {{shard_name}}",
456
          "refId": "A"
457
        }
458
      ],
459
      "title": "Active Tombstone Cleanup Threads",
460
      "type": "timeseries"
461
    },
462
    {
463
      "datasource": {
464
        "type": "prometheus",
465
        "uid": "Prometheus"
466
      },
467
      "fieldConfig": {
468
        "defaults": {
469
          "color": {
470
            "mode": "thresholds"
471
          },
472
          "mappings": [],
473
          "thresholds": {
474
            "mode": "absolute",
475
            "steps": [
476
              {
477
                "color": "green",
478
                "value": null
479
              },
480
              {
481
                "color": "red",
482
                "value": 80
483
              }
484
            ]
485
          }
486
        },
487
        "overrides": []
488
      },
489
      "gridPos": {
490
        "h": 8,
491
        "w": 3,
492
        "x": 18,
493
        "y": 9
494
      },
495
      "id": 15,
496
      "options": {
497
        "orientation": "auto",
498
        "reduceOptions": {
499
          "calcs": [
500
            "lastNotNull"
501
          ],
502
          "fields": "",
503
          "values": false
504
        },
505
        "showThresholdLabels": false,
506
        "showThresholdMarkers": true
507
      },
508
      "pluginVersion": "8.3.3",
509
      "targets": [
510
        {
511
          "datasource": {
512
            "type": "prometheus",
513
            "uid": "Prometheus"
514
          },
515
          "exemplar": true,
516
          "expr": "sum(vector_index_tombstone_cleanup_threads{})",
517
          "interval": "",
518
          "legendFormat": "",
519
          "refId": "A"
520
        }
521
      ],
522
      "title": "Active Tombstone Cleanup Threads",
523
      "type": "gauge"
524
    },
525
    {
526
      "datasource": {
527
        "type": "prometheus",
528
        "uid": "Prometheus"
529
      },
530
      "description": "If a node had an outgoing connection to another node that received a tombstone it must be reassigned.",
531
      "fieldConfig": {
532
        "defaults": {
533
          "color": {
534
            "mode": "thresholds"
535
          },
536
          "mappings": [],
537
          "thresholds": {
538
            "mode": "absolute",
539
            "steps": [
540
              {
541
                "color": "green",
542
                "value": null
543
              }
544
            ]
545
          }
546
        },
547
        "overrides": []
548
      },
549
      "gridPos": {
550
        "h": 8,
551
        "w": 3,
552
        "x": 21,
553
        "y": 9
554
      },
555
      "id": 17,
556
      "options": {
557
        "colorMode": "value",
558
        "graphMode": "area",
559
        "justifyMode": "auto",
560
        "orientation": "auto",
561
        "reduceOptions": {
562
          "calcs": [
563
            "lastNotNull"
564
          ],
565
          "fields": "",
566
          "values": false
567
        },
568
        "textMode": "auto"
569
      },
570
      "pluginVersion": "8.3.3",
571
      "targets": [
572
        {
573
          "datasource": {
574
            "type": "prometheus",
575
            "uid": "Prometheus"
576
          },
577
          "exemplar": true,
578
          "expr": "sum(vector_index_tombstone_cleaned{})",
579
          "interval": "",
580
          "legendFormat": "",
581
          "refId": "A"
582
        }
583
      ],
584
      "title": "Reassigned Nodes",
585
      "type": "stat"
586
    },
587
    {
588
      "datasource": {
589
        "type": "prometheus",
590
        "uid": "Prometheus"
591
      },
592
      "description": "Count of active segments across all classes and shards, shown by type.",
593
      "fieldConfig": {
594
        "defaults": {
595
          "color": {
596
            "mode": "palette-classic"
597
          },
598
          "custom": {
599
            "axisLabel": "",
600
            "axisPlacement": "auto",
601
            "barAlignment": 0,
602
            "drawStyle": "line",
603
            "fillOpacity": 0,
604
            "gradientMode": "none",
605
            "hideFrom": {
606
              "legend": false,
607
              "tooltip": false,
608
              "viz": false
609
            },
610
            "lineInterpolation": "linear",
611
            "lineStyle": {
612
              "fill": "solid"
613
            },
614
            "lineWidth": 1,
615
            "pointSize": 9,
616
            "scaleDistribution": {
617
              "type": "linear"
618
            },
619
            "showPoints": "auto",
620
            "spanNulls": false,
621
            "stacking": {
622
              "group": "A",
623
              "mode": "normal"
624
            },
625
            "thresholdsStyle": {
626
              "mode": "off"
627
            }
628
          },
629
          "mappings": [],
630
          "thresholds": {
631
            "mode": "absolute",
632
            "steps": [
633
              {
634
                "color": "green",
635
                "value": null
636
              },
637
              {
638
                "color": "red",
639
                "value": 80
640
              }
641
            ]
642
          }
643
        },
644
        "overrides": []
645
      },
646
      "gridPos": {
647
        "h": 7,
648
        "w": 6,
649
        "x": 0,
650
        "y": 18
651
      },
652
      "id": 7,
653
      "interval": "1",
654
      "options": {
655
        "legend": {
656
          "calcs": [],
657
          "displayMode": "list",
658
          "placement": "bottom"
659
        },
660
        "tooltip": {
661
          "mode": "single"
662
        }
663
      },
664
      "targets": [
665
        {
666
          "datasource": {
667
            "type": "prometheus",
668
            "uid": "Prometheus"
669
          },
670
          "exemplar": true,
671
          "expr": "sum(lsm_active_segments{strategy=\"replace\"})",
672
          "interval": "",
673
          "legendFormat": "Total \"Replace\" (Object Storage)",
674
          "refId": "A"
675
        },
676
        {
677
          "datasource": {
678
            "type": "prometheus",
679
            "uid": "Prometheus"
680
          },
681
          "exemplar": true,
682
          "expr": "sum(lsm_active_segments{strategy=\"mapcollection\"})",
683
          "hide": false,
684
          "interval": "",
685
          "legendFormat": "Total \"Map\" (Inverted storage with term frequency)",
686
          "refId": "B"
687
        },
688
        {
689
          "datasource": {
690
            "type": "prometheus",
691
            "uid": "Prometheus"
692
          },
693
          "exemplar": true,
694
          "expr": "sum(lsm_active_segments{strategy=\"setcollection\"})",
695
          "hide": false,
696
          "interval": "",
697
          "legendFormat": "Total \"Set\" (Inverted storage without frequency)",
698
          "refId": "C"
699
        }
700
      ],
701
      "title": "Active LSM Segments (summed by type)",
702
      "type": "timeseries"
703
    },
704
    {
705
      "datasource": {
706
        "type": "prometheus",
707
        "uid": "Prometheus"
708
      },
709
      "fieldConfig": {
710
        "defaults": {
711
          "color": {
712
            "mode": "thresholds"
713
          },
714
          "mappings": [],
715
          "thresholds": {
716
            "mode": "absolute",
717
            "steps": [
718
              {
719
                "color": "text",
720
                "value": null
721
              }
722
            ]
723
          }
724
        },
725
        "overrides": []
726
      },
727
      "gridPos": {
728
        "h": 7,
729
        "w": 6,
730
        "x": 6,
731
        "y": 18
732
      },
733
      "id": 19,
734
      "options": {
735
        "colorMode": "value",
736
        "graphMode": "area",
737
        "justifyMode": "center",
738
        "orientation": "auto",
739
        "reduceOptions": {
740
          "calcs": [
741
            "lastNotNull"
742
          ],
743
          "fields": "",
744
          "values": false
745
        },
746
        "textMode": "auto"
747
      },
748
      "pluginVersion": "8.3.3",
749
      "targets": [
750
        {
751
          "datasource": {
752
            "type": "prometheus",
753
            "uid": "Prometheus"
754
          },
755
          "exemplar": true,
756
          "expr": "sum(vector_index_operations{operation=\"create\"})",
757
          "interval": "",
758
          "legendFormat": "Vectors Inserted",
759
          "refId": "A"
760
        },
761
        {
762
          "datasource": {
763
            "type": "prometheus",
764
            "uid": "Prometheus"
765
          },
766
          "exemplar": true,
767
          "expr": "sum(vector_index_operations{operation=\"delete\"})",
768
          "hide": false,
769
          "interval": "",
770
          "legendFormat": "Vectors Deleted",
771
          "refId": "B"
772
        },
773
        {
774
          "datasource": {
775
            "type": "prometheus",
776
            "uid": "Prometheus"
777
          },
778
          "exemplar": false,
779
          "expr": "sum(vector_index_operations{operation=\"create\"}) - ignoring(operation) sum(vector_index_operations{operation=\"delete\"})",
780
          "hide": false,
781
          "instant": false,
782
          "interval": "",
783
          "legendFormat": "Net Vectors",
784
          "refId": "C"
785
        }
786
      ],
787
      "title": "Vector Index Statistics",
788
      "type": "stat"
789
    },
790
    {
791
      "datasource": {
792
        "type": "prometheus",
793
        "uid": "Prometheus"
794
      },
795
      "fieldConfig": {
796
        "defaults": {
797
          "color": {
798
            "mode": "palette-classic"
799
          },
800
          "custom": {
801
            "axisLabel": "",
802
            "axisPlacement": "auto",
803
            "barAlignment": 0,
804
            "drawStyle": "line",
805
            "fillOpacity": 0,
806
            "gradientMode": "none",
807
            "hideFrom": {
808
              "legend": false,
809
              "tooltip": false,
810
              "viz": false
811
            },
812
            "lineInterpolation": "linear",
813
            "lineWidth": 1,
814
            "pointSize": 5,
815
            "scaleDistribution": {
816
              "type": "linear"
817
            },
818
            "showPoints": "auto",
819
            "spanNulls": false,
820
            "stacking": {
821
              "group": "A",
822
              "mode": "none"
823
            },
824
            "thresholdsStyle": {
825
              "mode": "off"
826
            }
827
          },
828
          "mappings": [],
829
          "thresholds": {
830
            "mode": "absolute",
831
            "steps": [
832
              {
833
                "color": "green",
834
                "value": null
835
              },
836
              {
837
                "color": "red",
838
                "value": 80
839
              }
840
            ]
841
          },
842
          "unit": "decbytes"
843
        },
844
        "overrides": []
845
      },
846
      "gridPos": {
847
        "h": 8,
848
        "w": 12,
849
        "x": 0,
850
        "y": 25
851
      },
852
      "id": 4,
853
      "options": {
854
        "legend": {
855
          "calcs": [],
856
          "displayMode": "list",
857
          "placement": "bottom"
858
        },
859
        "tooltip": {
860
          "mode": "single"
861
        }
862
      },
863
      "targets": [
864
        {
865
          "datasource": {
866
            "type": "prometheus",
867
            "uid": "Prometheus"
868
          },
869
          "exemplar": true,
870
          "expr": "go_memstats_heap_inuse_bytes{}",
871
          "interval": "",
872
          "legendFormat": "",
873
          "refId": "A"
874
        }
875
      ],
876
      "title": "Heap in use (bytes)",
877
      "type": "timeseries"
878
    }
879
  ],
880
  "refresh": "5s",
881
  "schemaVersion": 34,
882
  "style": "dark",
883
  "tags": [],
884
  "templating": {
885
    "list": []
886
  },
887
  "time": {
888
    "from": "now-15m",
889
    "to": "now"
890
  },
891
  "timepicker": {},
892
  "timezone": "",
893
  "title": "Importing Data into Weaviate",
894
  "uid": "n_WGWQ1nk3",
895
  "version": 1,
896
  "weekStart": ""
897
}
898