claude-botandClaude Opus 4.8
c7e9fc8bb6
The dashboard was entirely dead: every panel queried postgres_exporter metrics (pg_up, pg_stat_activity_count, ...) that have never existed in this Prometheus. Postgres runs as four CloudNativePG clusters (ak-pg, im-pg, gf-pg, pprl-pg) whose built-in exporter on :9187 was not scraped at all (enablePodMonitor is false and no PodMonitor existed). - add PodMonitors for the four CNPG clusters in monitoring-rules, labelled release=kube-prometheus-stack so the Prometheus podMonitorSelector actually picks them up (CNPG's own enablePodMonitor would generate unlabelled PodMonitors that this Prometheus ignores) - rewrite all dashboard queries to the cnpg_* metrics the exporter actually exposes (verified against the pods' /metrics endpoints) - dedupe multi-instance clusters (max by namespace/datname for sizes, primary-only filter via cnpg_pg_replication_in_recovery == 0 for per-database counters) - replace the Active Locks stat (no lock metric in the CNPG default queries) with Max Replication Lag - instant queries on all stat panels so dead instances show No data instead of a stale last value Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
222 lines
15 KiB
JSON
222 lines
15 KiB
JSON
{
|
|
"annotations": {"list": []},
|
|
"description": "PostgreSQL (CloudNativePG) database monitoring: ak-pg (authentik), im-pg (immich), gf-pg (grafana), pprl-pg (paperless)",
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {"color": {"mode": "thresholds"}, "mappings": [{"options": {"1": {"color": "green", "index": 0, "text": "Up"}}, "type": "value"}, {"options": {"0": {"color": "red", "index": 1, "text": "Down"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
|
"id": 1,
|
|
"options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "value_and_name"},
|
|
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "cnpg_collector_up", "instant": true, "legendFormat": "{{pod}}", "refId": "A"}],
|
|
"title": "Instances",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 200}, {"color": "red", "value": 320}]}},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 6, "y": 0},
|
|
"id": 2,
|
|
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
|
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cnpg_backends_total)", "instant": true, "refId": "A"}],
|
|
"title": "Connections",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": null}]}, "unit": "bytes"},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
|
|
"id": 3,
|
|
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
|
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(max by (namespace, datname) (cnpg_pg_database_size_bytes{datname!~\"template.*|postgres\"}))", "instant": true, "refId": "A"}],
|
|
"title": "Total Database Size",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 10}, {"color": "red", "value": 60}]}, "unit": "s"},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 4, "x": 15, "y": 0},
|
|
"id": 4,
|
|
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
|
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(cnpg_pg_replication_lag)", "instant": true, "refId": "A"}],
|
|
"title": "Max Replication Lag",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "ops"},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 4, "w": 5, "x": 19, "y": 0},
|
|
"id": 5,
|
|
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
|
|
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum((rate(cnpg_pg_stat_database_xact_commit[5m]) + rate(cnpg_pg_stat_database_xact_rollback[5m])) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "instant": true, "refId": "A"}],
|
|
"title": "Transactions/sec",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
|
|
"mappings": [],
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
|
|
"id": 6,
|
|
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (namespace, state) (cnpg_backends_total)", "legendFormat": "{{namespace}} · {{state}}", "refId": "A"}
|
|
],
|
|
"title": "Connection States",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
|
|
"mappings": [],
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
|
|
"id": 7,
|
|
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (namespace, datname) (rate(cnpg_pg_stat_database_xact_commit{datname!~\"template.*|postgres\", datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "Commits · {{datname}}", "refId": "A"},
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (namespace, datname) (rate(cnpg_pg_stat_database_xact_rollback{datname!~\"template.*|postgres\", datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "Rollbacks · {{datname}}", "refId": "B"}
|
|
],
|
|
"title": "Transaction Rate",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
|
|
"mappings": [],
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
|
|
"max": 100,
|
|
"min": 0,
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
|
|
"id": 8,
|
|
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 * sum by (namespace, datname) (rate(cnpg_pg_stat_database_blks_hit{datname!~\"template.*|postgres\", datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0)) / sum by (namespace, datname) ((rate(cnpg_pg_stat_database_blks_hit{datname!~\"template.*|postgres\", datname!=\"\"}[5m]) + rate(cnpg_pg_stat_database_blks_read{datname!~\"template.*|postgres\", datname!=\"\"}[5m])) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "{{datname}}", "refId": "A"}
|
|
],
|
|
"title": "Cache Hit Ratio",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
|
|
"mappings": [],
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
|
|
"id": 9,
|
|
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (namespace, datname) (cnpg_pg_database_size_bytes{datname!~\"template.*|postgres\"})", "legendFormat": "{{datname}}", "refId": "A"}
|
|
],
|
|
"title": "Database Sizes",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
|
|
"mappings": [],
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
|
|
"id": 10,
|
|
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cnpg_pg_stat_database_tup_fetched{datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "Fetched", "refId": "A"},
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cnpg_pg_stat_database_tup_inserted{datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "Inserted", "refId": "B"},
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cnpg_pg_stat_database_tup_updated{datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "Updated", "refId": "C"},
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cnpg_pg_stat_database_tup_deleted{datname!=\"\"}[5m]) and on (namespace, pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "Deleted", "refId": "D"}
|
|
],
|
|
"title": "Tuple Operations",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {"mode": "palette-classic"},
|
|
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
|
|
"mappings": [],
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
|
|
"id": 11,
|
|
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (namespace) (cnpg_pg_replication_lag)", "legendFormat": "Replication lag · {{namespace}}", "refId": "A"},
|
|
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (namespace) (cnpg_backends_max_tx_duration_seconds)", "legendFormat": "Longest tx · {{namespace}}", "refId": "B"}
|
|
],
|
|
"title": "Replication & Transaction Health",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 38,
|
|
"style": "dark",
|
|
"tags": ["homelab", "postgresql", "database"],
|
|
"templating": {"list": []},
|
|
"time": {"from": "now-1h", "to": "now"},
|
|
"timepicker": {},
|
|
"timezone": "",
|
|
"title": "PostgreSQL Overview",
|
|
"uid": "postgresql-overview",
|
|
"version": 1,
|
|
"weekStart": ""
|
|
}
|