fix: [alert] LokiCanaryMissingEntries (cluster) (#844 )

2026-09-16 07:45:29 +00:00
1 changed files with 14 additions and 3 deletions
@@ -47,10 +47,21 @@ spec:
        - alert: LokiCanaryMissingEntries
          # loki-canary writes test logs and reads them back. Missing
          # entries mean the ingest → store → query path lost data.
-          expr: sum(increase(loki_canary_missing_entries_total[10m])) > 0
+          #
+          # Aggregated per pod, not cluster-wide: the canary is a DaemonSet
+          # and pushes directly to Loki, so a single wedged node stalls only
+          # its own canary. A bare sum() turned that into an unlabelled
+          # "log data loss" alert that named neither the node nor Loki.
+          # Grouping by pod keeps genuine Loki data loss loud (every canary
+          # fires) while a single-node stall is self-describing.
+          expr: sum by (pod) (increase(loki_canary_missing_entries_total[10m])) > 0
          for: 10m
          labels:
            severity: warning
          annotations:
-            summary: "Loki canary detected missing log entries"
-            description: "loki-canary reported entries it wrote were not returned by the read path in the last 10m — log data loss."
+            summary: "Loki canary {{ $labels.pod }} detected missing log entries"
+            description: >-
+              loki-canary {{ $labels.pod }} reported entries it wrote were not
+              returned by the read path in the last 10m. If only this canary is
+              affected, suspect its node (blocked I/O, resource starvation)
+              rather than Loki; if all canaries are affected, suspect Loki.