claude-botandClaude Opus 5.5
59ebc125fa
bird and netbird-service-sync run images from Gitea's registry with no copy on ghcr.io. In the infrastructure Kustomization they started long before the restored Gitea served them and sat in ImagePullBackOff (restore rehearsal runs 27, 28). infrastructure cannot depend on gitea, which depends on it, so both move to a new Kustomization, infrastructure-inhouse, that depends on infrastructure, infrastructure-crds and gitea. Their namespace stays in infrastructure. e2e-test.sh checks the new Kustomization for Ready. Images published to ghcr.io too have a fallback in the nodes' registry mirror and need no ordering; multus cni-plugins stays in infrastructure. The secret-rotator and backup-verify CronJobs also use Gitea images but only run on schedule; they are left in place. bird fills routing table 47, which exists only where the site patch routes pod traffic into it, and renders its config from node annotations only that site sets. On a site without it the template rendered <no value> and the init container crash-looped (run 28). The DaemonSet now selects on net.renner.uno/pod-table=47, which site-homelab.yaml puts on its nodes. Production rollout: 1. The pod-table label must be on the nodes (talosctl patch, no reboot) before the DaemonSet change reconciles, or bird is removed from every node and pod traffic falls through to the main table. 2. infrastructure must run with prune: false (the stack's first PR) when this merges. It then drops bird and netbird-service-sync from its inventory without deleting them, and infrastructure-inhouse adopts them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
659 lines
30 KiB
Bash
Executable File
659 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# End-to-end smoke test for the homelab K8s cluster.
|
|
# Exits non-zero if any check fails. Prints a summary at the end.
|
|
#
|
|
# Assumes kubeconfig already loaded at ~/.kube/config and the runner can
|
|
# reach the ingress LB IP (10.2.99.86) and the cluster API endpoint.
|
|
set -uo pipefail
|
|
|
|
LB_IP="10.2.99.86"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
FAILURES=()
|
|
|
|
check() {
|
|
local name="$1"; shift
|
|
printf " %-60s " "$name"
|
|
if "$@" >/tmp/check.out 2>&1; then
|
|
echo "OK"
|
|
PASS=$((PASS + 1))
|
|
else
|
|
echo "FAIL"
|
|
FAILURES+=("$name -- $(tail -4 /tmp/check.out | tr '\n' ' ')")
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
# curl helper: exact status-code match. -k because we accept staging LE
|
|
# certs (not in the public bundle); the dedicated TLS check below
|
|
# validates the cert issuer.
|
|
http_code() {
|
|
local url="$1" expect="$2" host code
|
|
host=$(echo "$url" | awk -F/ '{print $3}')
|
|
code=$(curl -sk --resolve "$host:443:$LB_IP" -o /dev/null -w '%{http_code}' \
|
|
--max-time 10 "$url")
|
|
echo "got=$code expect=$expect"
|
|
test "$code" = "$expect"
|
|
}
|
|
|
|
# curl helper: accept any 2xx/3xx/4xx (not 5xx / not 000).
|
|
http_live() {
|
|
local url="$1" host code
|
|
host=$(echo "$url" | awk -F/ '{print $3}')
|
|
code=$(curl -sk --resolve "$host:443:$LB_IP" -o /dev/null -w '%{http_code}' \
|
|
--max-time 10 "$url")
|
|
echo "got=$code"
|
|
[[ "$code" =~ ^[234][0-9][0-9]$ ]]
|
|
}
|
|
|
|
http_body() {
|
|
local url="$1" needle="$2" host raw body code
|
|
host=$(echo "$url" | awk -F/ '{print $3}')
|
|
# Retry 3x with backoff: a transient TLS/conn blip from the runner to
|
|
# the LB looks identical to a bad body when curl is silenced. Logs
|
|
# `code=` and `body=` per attempt so failure summaries are diagnostic
|
|
# instead of a bare `-- ` (the previous version piped through `grep -q`,
|
|
# which produces no stdout on failure).
|
|
for attempt in 1 2 3; do
|
|
raw=$(curl -sk --resolve "$host:443:$LB_IP" --max-time 15 \
|
|
-w '\n__HTTP_CODE__%{http_code}' "$url" 2>&1)
|
|
code=${raw##*__HTTP_CODE__}
|
|
body=${raw%__HTTP_CODE__*}
|
|
body=${body%$'\n'}
|
|
echo "attempt=$attempt code=$code body=$body"
|
|
echo "$body" | grep -q "$needle" && return 0
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Force-delete Released PVs whose claimRef matches a pattern. The ceph-block
|
|
# and ceph-filesystem StorageClasses ship with reclaimPolicy: Retain so that
|
|
# production PVC deletions can be undone manually. The e2e test has the
|
|
# opposite need: every PVC it creates is throwaway and MUST not survive the
|
|
# run. Without this each invocation leaks ~2 Released PVs that pin CephFS
|
|
# subvolume / RBD image data on disk indefinitely.
|
|
purge_released_pvs() {
|
|
local pattern="$1" # regex applied to claimRef.namespace + "/" + claimRef.name
|
|
local pv pvs
|
|
# No jq here: the ci-k8s image doesn't ship it, and the original jq
|
|
# version silently produced an empty list for months, leaking Released
|
|
# PVs (and their CephFS/RBD data) on every run — issue #262.
|
|
pvs=$(kubectl get pv -o jsonpath='{range .items[?(@.status.phase=="Released")]}{.metadata.name}{" "}{.spec.claimRef.namespace}{"/"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null \
|
|
| awk -v p="$pattern" '$2 ~ p {print $1}')
|
|
for pv in $pvs; do
|
|
kubectl patch pv "$pv" --type=merge \
|
|
-p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null 2>&1 || true
|
|
kubectl delete pv "$pv" --wait=false --ignore-not-found >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
# Verify ingress serves a cert-manager-issued LE cert (prod or staging),
|
|
# not the ingress-nginx default fake cert. Catches cert-manager failures
|
|
# without requiring a specific trust chain, so the same test works in
|
|
# both dev (staging issuer) and prod.
|
|
tls_valid() {
|
|
local host="$1" issuer
|
|
issuer=$(echo | openssl s_client -servername "$host" -connect "$LB_IP:443" 2>/dev/null \
|
|
| openssl x509 -noout -issuer 2>/dev/null | sed 's/^issuer=//')
|
|
echo "issuer=$issuer"
|
|
[[ "$issuer" == *"Let's Encrypt"* ]]
|
|
}
|
|
|
|
echo "=== Cluster ==="
|
|
check "3 nodes Ready" \
|
|
bash -c 'test "$(kubectl get nodes --no-headers | grep -c " Ready ")" -eq 3'
|
|
|
|
echo "=== Flux Kustomizations ==="
|
|
for ks in flux-system infrastructure infrastructure-crds infrastructure-ceph-cluster infrastructure-inhouse \
|
|
influxdb \
|
|
apps authentik paperless audiobookshelf woodpecker immich; do
|
|
check "kustomization/$ks Ready" \
|
|
bash -c "kubectl -n flux-system get kustomization $ks \
|
|
-o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -q '^True\$'"
|
|
done
|
|
|
|
echo "=== HelmReleases ==="
|
|
check "all HelmReleases Ready" bash -c '
|
|
bad=$(kubectl get helmrelease -A -o jsonpath="{range .items[*]}{.metadata.namespace}/{.metadata.name}:{.status.conditions[?(@.type==\"Ready\")].status}{\"\n\"}{end}" \
|
|
| grep -v ":True$" || true)
|
|
[[ -z "$bad" ]] || { echo "$bad"; exit 1; }
|
|
'
|
|
|
|
echo "=== Pods ==="
|
|
check "no failing pods" bash -c '
|
|
# woodpecker-ci is excluded: the runner pod executing THIS e2e run (and
|
|
# any parallel build) is legitimately Pending/ContainerCreating for a
|
|
# few seconds and made the check flap (issue #262).
|
|
bad=$(kubectl get pods -A \
|
|
--field-selector=status.phase!=Running,status.phase!=Succeeded \
|
|
--no-headers 2>/dev/null | grep -v "^woodpecker-ci " || true)
|
|
[[ -z "$bad" ]] || { echo "$bad"; exit 1; }
|
|
'
|
|
|
|
echo "=== Ceph ==="
|
|
check "CephCluster phase=Ready" \
|
|
bash -c 'kubectl -n rook-ceph get cephcluster rook-ceph \
|
|
-o jsonpath="{.status.phase}" | grep -q Ready'
|
|
check "3 OSDs Running" \
|
|
bash -c 'test "$(kubectl -n rook-ceph get pod -l app=rook-ceph-osd \
|
|
--field-selector=status.phase=Running --no-headers | wc -l)" -eq 3'
|
|
check "CephObjectStore migration Ready" \
|
|
bash -c 'kubectl -n rook-ceph get cephobjectstore migration \
|
|
-o jsonpath="{.status.phase}" | grep -q Ready'
|
|
|
|
echo "=== Storage provisioning (ceph-block) ==="
|
|
kubectl -n default delete pod e2e-writer --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
|
kubectl -n default delete pvc e2e-pvc --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
|
cat <<'EOF' | kubectl apply -f - >/dev/null
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: { name: e2e-pvc, namespace: default }
|
|
spec:
|
|
accessModes: [ReadWriteOnce]
|
|
resources: { requests: { storage: 1Gi } }
|
|
# Throwaway PVC — use the Delete-reclaim class so its RBD image is removed on
|
|
# deletion instead of leaking a Released PV (was ~1 image/day; issue #262).
|
|
storageClassName: ceph-block-ephemeral
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: { name: e2e-writer, namespace: default }
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
fsGroup: 65532
|
|
seccompProfile: { type: RuntimeDefault }
|
|
containers:
|
|
- name: writer
|
|
image: alpine:3.21
|
|
command: ["sh", "-c", "echo e2e-ok-$(date +%s) > /d/t && grep -q e2e-ok /d/t"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: { drop: ["ALL"] }
|
|
volumeMounts: [{ name: d, mountPath: /d }]
|
|
volumes: [{ name: d, persistentVolumeClaim: { claimName: e2e-pvc } }]
|
|
EOF
|
|
check "ceph-block PVC provisions and pod writes+reads" bash -c '
|
|
for i in $(seq 1 60); do
|
|
phase=$(kubectl -n default get pod e2e-writer -o jsonpath="{.status.phase}" 2>/dev/null)
|
|
[[ "$phase" == "Succeeded" ]] && exit 0
|
|
[[ "$phase" == "Failed" ]] && { kubectl -n default logs e2e-writer; exit 1; }
|
|
sleep 2
|
|
done
|
|
kubectl -n default describe pod e2e-writer
|
|
exit 1
|
|
'
|
|
kubectl -n default delete pod e2e-writer --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
|
kubectl -n default delete pvc e2e-pvc --ignore-not-found --wait=true >/dev/null 2>&1 || true
|
|
# e2e-pvc now uses ceph-block-ephemeral (Delete reclaim), so its PV is removed
|
|
# automatically and this normally finds nothing. Kept as a one-time safety net
|
|
# to sweep any Retain-era e2e-pvc PV left over from before the ephemeral SC.
|
|
purge_released_pvs '^default/e2e-pvc$'
|
|
|
|
echo "=== Velero backup + restore roundtrip ==="
|
|
V_NS=velero-e2e
|
|
V_BACKUP="e2e-$(date -u +%s)"
|
|
V_RESTORE="${V_BACKUP}-restore"
|
|
V_MARKER="velero-e2e-$(date -u +%s)"
|
|
|
|
kubectl delete ns $V_NS --wait=true --timeout=60s --ignore-not-found >/dev/null 2>&1 || true
|
|
cat <<EOF | kubectl apply -f - >/dev/null
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata: { name: $V_NS }
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata: { name: data, namespace: $V_NS }
|
|
spec:
|
|
# CephFS RWX, not RBD RWO. Velero CSI snapshot data movement on RBD
|
|
# fails when the source pod still has the volume RW-mounted on the
|
|
# same node ("cannot mount /dev/rbdN read-only" — the krbd device
|
|
# can't be active in two modes at once). CephFS doesn't share that
|
|
# limitation; concurrent RO snapshot mount + RW source mount work.
|
|
# The test still exercises the same CSI data-movement code path.
|
|
accessModes: [ReadWriteMany]
|
|
resources: { requests: { storage: 1Gi } }
|
|
storageClassName: ceph-filesystem
|
|
---
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: { name: writer, namespace: $V_NS }
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
fsGroup: 65532
|
|
seccompProfile: { type: RuntimeDefault }
|
|
containers:
|
|
- name: writer
|
|
image: alpine:3.21
|
|
command: ["sh", "-c", "echo ${V_MARKER} > /d/m && sleep 3600"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: { drop: ["ALL"] }
|
|
volumeMounts: [{ name: d, mountPath: /d }]
|
|
volumes: [{ name: d, persistentVolumeClaim: { claimName: data } }]
|
|
EOF
|
|
|
|
check "BSL default is Available" \
|
|
bash -c 'kubectl -n velero get backupstoragelocation default \
|
|
-o jsonpath="{.status.phase}" | grep -q Available'
|
|
check "node-agent DaemonSet has 3 pods ready" \
|
|
bash -c 'test "$(kubectl -n velero get ds node-agent -o jsonpath={.status.numberReady})" -ge 3'
|
|
check "VolumeSnapshotClass ceph-filesystem-snapclass present" \
|
|
bash -c 'kubectl get volumesnapshotclass ceph-filesystem-snapclass \
|
|
-o jsonpath="{.metadata.labels.velero\.io/csi-volumesnapshot-class}" | grep -q true'
|
|
check "velero-e2e: writer pod ready (pre-backup)" \
|
|
bash -c "kubectl -n $V_NS wait --for=condition=ready pod/writer --timeout=120s"
|
|
check "velero-e2e: marker readable (pre-backup)" \
|
|
bash -c "kubectl -n $V_NS exec writer -- cat /d/m | grep -q ${V_MARKER}"
|
|
|
|
cat <<EOF | kubectl apply -f - >/dev/null
|
|
apiVersion: velero.io/v1
|
|
kind: Backup
|
|
metadata: { name: $V_BACKUP, namespace: velero }
|
|
spec:
|
|
includedNamespaces: [$V_NS]
|
|
snapshotMoveData: true
|
|
defaultVolumesToFsBackup: false
|
|
ttl: 2h0m0s
|
|
storageLocation: default
|
|
EOF
|
|
|
|
# 120x15s = 30 min, not the old 60x5s = 5 min. node-agent has no
|
|
# loadConcurrency override, so Velero's default of one data path per node
|
|
# applies: while the restore-test sweep holds the slot, even this 1 GiB
|
|
# DataUpload sits in phase Prepared for well over 5 minutes. The old 300 s wait
|
|
# then let the teardown below delete the source namespace and the Backup
|
|
# mid-upload ("Cancel du under phase Prepared because it is being deleted"),
|
|
# leaving a PartiallyFailed Backup that looks like a real backup fault during
|
|
# triage (seen 2026-07-26 at 02:00 and 12:04, both inside a running sweep).
|
|
V_BACKUP_FAILED_BEFORE=$FAIL
|
|
check "velero-e2e: backup Completed (CSI data movement)" bash -c "
|
|
for _ in \$(seq 1 120); do
|
|
p=\$(kubectl -n velero get backup.velero.io $V_BACKUP -o jsonpath='{.status.phase}' 2>/dev/null)
|
|
[ \"\$p\" = 'Completed' ] && exit 0
|
|
echo \"\$p\" | grep -qE 'Failed|PartiallyFailed|FailedValidation' \
|
|
&& { kubectl -n velero get backup.velero.io $V_BACKUP -o yaml | tail -40; exit 1; }
|
|
sleep 15
|
|
done
|
|
exit 1
|
|
"
|
|
|
|
# Only run the destructive half if the backup actually completed. Tearing down
|
|
# the source namespace while the DataUpload is still in flight is what cancels
|
|
# it and manufactures the phantom PartiallyFailed Backup.
|
|
if [[ "$FAIL" -ne "$V_BACKUP_FAILED_BEFORE" ]]; then
|
|
echo " (skipping restore roundtrip: backup never completed — tearing down now"
|
|
echo " would cancel the in-flight DataUpload and leave a PartiallyFailed Backup)"
|
|
else
|
|
check "velero-e2e: backup has >=1 items" \
|
|
bash -c "test \"\$(kubectl -n velero get backup.velero.io $V_BACKUP \
|
|
-o jsonpath='{.status.progress.totalItems}')\" -ge 1"
|
|
|
|
kubectl delete ns $V_NS --wait=true --timeout=120s >/dev/null 2>&1 || true
|
|
|
|
cat <<EOF | kubectl apply -f - >/dev/null
|
|
apiVersion: velero.io/v1
|
|
kind: Restore
|
|
metadata: { name: $V_RESTORE, namespace: velero }
|
|
spec:
|
|
backupName: $V_BACKUP
|
|
includedNamespaces: [$V_NS]
|
|
EOF
|
|
|
|
check "velero-e2e: restore Completed" bash -c "
|
|
for _ in \$(seq 1 60); do
|
|
p=\$(kubectl -n velero get restore.velero.io $V_RESTORE -o jsonpath='{.status.phase}' 2>/dev/null)
|
|
[ \"\$p\" = 'Completed' ] && exit 0
|
|
echo \"\$p\" | grep -qE 'Failed|PartiallyFailed|FailedValidation' \
|
|
&& { kubectl -n velero get restore.velero.io $V_RESTORE -o yaml | tail -40; exit 1; }
|
|
sleep 5
|
|
done
|
|
exit 1
|
|
"
|
|
check "velero-e2e: restored writer pod ready" \
|
|
bash -c "kubectl -n $V_NS wait --for=condition=ready pod/writer --timeout=180s"
|
|
check "velero-e2e: marker survived roundtrip" \
|
|
bash -c "kubectl -n $V_NS exec writer -- cat /d/m | grep -q ${V_MARKER}"
|
|
fi
|
|
|
|
# Only delete the Backup if it actually completed. Deleting one whose
|
|
# DataUpload is still in flight is the very thing that cancels the upload and
|
|
# manufactures a phantom PartiallyFailed Backup — the same race the guard above
|
|
# avoids, just relocated to cleanup. The Backup carries ttl: 2h0m0s, so leaving
|
|
# it costs nothing: Velero's GC reaps it either way.
|
|
if [[ "$FAIL" -eq "$V_BACKUP_FAILED_BEFORE" ]]; then
|
|
kubectl -n velero delete backup.velero.io $V_BACKUP --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
|
else
|
|
echo " (leaving backup $V_BACKUP to its 2h TTL: deleting it now could cancel an in-flight DataUpload)"
|
|
fi
|
|
kubectl -n velero delete restore.velero.io $V_RESTORE --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
|
kubectl delete ns $V_NS --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || true
|
|
# Each run creates the velero-e2e/data PVC twice (once for the backup source
|
|
# pod, once when the restore re-provisions it), so both PVs need purging.
|
|
purge_released_pvs "^${V_NS}/.*"
|
|
|
|
echo "=== CNPG Barman Cloud backup wiring ==="
|
|
check "barman-cloud plugin pod Ready in database ns" \
|
|
bash -c 'test "$(kubectl -n database get pod -l app=barman-cloud -o jsonpath={.items[0].status.containerStatuses[0].ready})" = "true"'
|
|
for os in authentik/ak-pg-store paperless/pprl-pg-store; do
|
|
ns=${os%%/*}; name=${os##*/}
|
|
check "ObjectStore $os exists" \
|
|
bash -c "kubectl -n $ns get objectstore $name -o jsonpath='{.metadata.name}' | grep -q ."
|
|
done
|
|
for pair in authentik/ak-pg paperless/pprl-pg; do
|
|
ns=${pair%%/*}; name=${pair##*/}
|
|
check "Cluster $pair has ContinuousArchiving=True" \
|
|
bash -c "kubectl -n $ns get cluster $name \
|
|
-o jsonpath='{.status.conditions[?(@.type==\"ContinuousArchiving\")].status}' | grep -q '^True\$'"
|
|
done
|
|
for pair in authentik/ak-pg-daily paperless/pprl-pg-daily; do
|
|
ns=${pair%%/*}; name=${pair##*/}
|
|
check "ScheduledBackup $pair has produced >=1 completed Backup" \
|
|
bash -c "test \"\$(kubectl -n $ns get backup.postgresql.cnpg.io \
|
|
-o jsonpath='{range .items[?(@.status.phase==\"completed\")]}x{end}' | wc -c)\" -ge 1"
|
|
done
|
|
|
|
echo "=== Databases (CloudNativePG) ==="
|
|
check "ak-pg: 2 ready instances" \
|
|
bash -c 'test "$(kubectl -n authentik get cluster ak-pg -o jsonpath={.status.readyInstances})" = 2'
|
|
check "pprl-pg: 2 ready instances" \
|
|
bash -c 'test "$(kubectl -n paperless get cluster pprl-pg -o jsonpath={.status.readyInstances})" = 2'
|
|
|
|
echo "=== cert-manager ClusterIssuers ==="
|
|
for ci in letsencrypt-prod letsencrypt-staging letsencrypt-cluster; do
|
|
check "clusterissuer/$ci Ready" \
|
|
bash -c "kubectl get clusterissuer $ci \
|
|
-o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}' | grep -q '^True\$'"
|
|
done
|
|
|
|
echo "=== TLS certificate validation (ingress LB) ==="
|
|
for h in status.renner.uno auth.renner.uno \
|
|
dash.renner.uno audiobooks.renner.uno; do
|
|
check "TLS valid for $h" tls_valid "$h"
|
|
done
|
|
|
|
echo "=== App health endpoints ==="
|
|
check "gatus / returns 200" http_code "https://status.renner.uno/" 200
|
|
check "authentik live probe returns 200" http_code "https://auth.renner.uno/-/health/live/" 200
|
|
check "authentik ready probe returns 200" http_code "https://auth.renner.uno/-/health/ready/" 200
|
|
check "grafana /api/health says database=ok" http_body "https://dash.renner.uno/api/health" '"database": "ok"'
|
|
check "audiobookshelf /healthcheck returns 200" http_code "https://audiobooks.renner.uno/healthcheck" 200
|
|
|
|
echo "=== External DNS (public resolution) ==="
|
|
check "status.renner.uno resolves via 1.1.1.1" \
|
|
bash -c 'dig +short +time=3 +tries=2 @1.1.1.1 status.renner.uno | grep -qE "[a-zA-Z0-9]"'
|
|
check "auth.renner.uno resolves via 1.1.1.1" \
|
|
bash -c 'dig +short +time=3 +tries=2 @1.1.1.1 auth.renner.uno | grep -qE "[a-zA-Z0-9]"'
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Functional checks below this line — exercise actual data, integrations,
|
|
# and observability flow rather than just probing readiness.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
echo "=== Data presence (post-restore) ==="
|
|
# Floors (>=) chosen to be well below current values so adding/removing real
|
|
# data doesn't break the test, while a fully empty restore would.
|
|
# CNPG instance pods are NOT stable names — the operator replaces instances
|
|
# on failover/maintenance (pprl-pg-1 no longer exists; issue #262). Resolve
|
|
# the current primary by role label instead of hardcoding a pod.
|
|
cnpg_primary() { # <namespace> <cluster>
|
|
kubectl -n "$1" get pod \
|
|
-l "cnpg.io/cluster=$2,cnpg.io/instanceRole=primary" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null
|
|
}
|
|
# Exported so the `bash -c '...'` subshells the `check` helper uses can call
|
|
# it (same mechanism as helper_curl below).
|
|
export -f cnpg_primary
|
|
check "paperless: documents_document rows >= 100" \
|
|
bash -c 'n=$(kubectl -n paperless exec "$(cnpg_primary paperless pprl-pg)" -c postgres -- \
|
|
psql -U postgres -d paperless -tAc "select count(*) from documents_document" 2>/dev/null \
|
|
| tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 100'
|
|
check "authentik: users >= 1" \
|
|
bash -c 'n=$(kubectl -n authentik exec "$(cnpg_primary authentik ak-pg)" -c postgres -- \
|
|
psql -U postgres -d authentik -tAc "select count(*) from authentik_core_user" 2>/dev/null \
|
|
| tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 1'
|
|
check "authentik: applications >= 1" \
|
|
bash -c 'n=$(kubectl -n authentik exec "$(cnpg_primary authentik ak-pg)" -c postgres -- \
|
|
psql -U postgres -d authentik -tAc "select count(*) from authentik_core_application" 2>/dev/null \
|
|
| tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 1'
|
|
check "audiobookshelf: /audiobooks has >= 50 entries" \
|
|
bash -c 'n=$(kubectl -n audiobookshelf exec deploy/audiobookshelf -c audiobookshelf -- \
|
|
sh -c "ls /audiobooks 2>/dev/null | wc -l"); echo "n=$n"; test "$n" -ge 50'
|
|
|
|
# ---- Provision a helper pod for in-cluster HTTP queries (Loki/Prom/AM) ----
|
|
# Hard-fail if the helper can't come up: silent failure here was the root cause
|
|
# of #730's empty observability bodies (kubectl exec against a missing/dying
|
|
# pod returns no stdout, so the per-check $out was empty and the failure tail
|
|
# was uninformative).
|
|
# The helper lives in the MONITORING namespace: its whole purpose is to curl
|
|
# Loki/Prometheus/Alertmanager, and monitoring's default-deny-ingress CNP
|
|
# only admits intra-namespace traffic (plus a short allow-list that `default`
|
|
# is not on — running it there timed out every obs check, issue #262).
|
|
kubectl -n monitoring delete pod e2e-helper --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true
|
|
cat <<'EOF' | kubectl apply -f - >/dev/null
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: { name: e2e-helper, namespace: monitoring }
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
seccompProfile: { type: RuntimeDefault }
|
|
containers:
|
|
- name: c
|
|
image: curlimages/curl:8.10.1
|
|
# 1800s = 30m. Long enough to outlive the rest of e2e even if the
|
|
# downstream checks hit their own timeouts.
|
|
command: ["sleep", "1800"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: { drop: ["ALL"] }
|
|
EOF
|
|
if ! kubectl -n monitoring wait --for=condition=ready pod/e2e-helper --timeout=180s; then
|
|
echo "ERROR: e2e-helper pod did not become ready in 180s. Diagnostic:"
|
|
kubectl -n monitoring describe pod e2e-helper || true
|
|
exit 1
|
|
fi
|
|
# Sanity check: confirm exec actually works before the obs section depends on it.
|
|
if ! kubectl -n monitoring exec e2e-helper -- echo "helper-alive" | grep -q helper-alive; then
|
|
echo "ERROR: kubectl exec into e2e-helper failed despite Ready status."
|
|
kubectl -n monitoring describe pod e2e-helper || true
|
|
exit 1
|
|
fi
|
|
|
|
# helper_curl: run a single curl inside e2e-helper, return body+code, retry on
|
|
# transient errors. Logs every attempt (HTTP code + body prefix) into the
|
|
# caller's /tmp/check.out so silent failures are impossible. Writes the final
|
|
# body to /tmp/helper_body for the caller to assert against.
|
|
# 2>&1 on the kubectl exec captures both kubectl errors AND curl errors so we
|
|
# can see *why* something failed (e.g. ConnectionRefused, NXDOMAIN, 502).
|
|
# Exported so the `bash -c '...'` subshells the `check` helper uses can call it.
|
|
helper_curl() {
|
|
local url="$1"
|
|
local out body code attempt
|
|
for attempt in 1 2 3 4 5; do
|
|
out=$(kubectl -n monitoring exec e2e-helper -- \
|
|
curl -sk --max-time 10 -w $'\n__CODE__:%{http_code}' "$url" 2>&1)
|
|
body=${out%$'\n'__CODE__:*}
|
|
code=${out##*__CODE__:}
|
|
printf 'attempt=%s code=%s body=%s\n' "$attempt" "$code" "${body:0:200}"
|
|
if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
|
|
printf '%s' "$body" > /tmp/helper_body
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
export -f helper_curl
|
|
|
|
echo "=== Observability data flow ==="
|
|
# Loki: K8s pods shipped log lines in the last 5 minutes. Proves Alloy → Loki
|
|
# ingestion is live, not just that Loki is running. K8s-side Alloy uses pod
|
|
# discovery so the labels are namespace/pod/container, not the `host` label
|
|
# that the LXC/VM Alloy stack uses. Query is namespace-scoped accordingly.
|
|
check "loki: K8s logs in last 5m > 0" bash -c '
|
|
helper_curl "http://loki.monitoring.svc:3100/loki/api/v1/query?query=sum(count_over_time(%7Bnamespace%3D~%22.%2B%22%7D%5B5m%5D))" || exit 1
|
|
grep -qE "\"value\":\[[0-9.]+,\"[1-9][0-9]*\"\]" /tmp/helper_body
|
|
'
|
|
|
|
# Prometheus: count of healthy scrape targets. Threshold low enough to absorb
|
|
# normal scaling noise (current baseline ~32), high enough that a broken Prom
|
|
# would obviously fail.
|
|
check "prometheus: count(up == 1) >= 25" bash -c '
|
|
helper_curl "http://kube-prometheus-stack-prometheus.monitoring.svc:9090/api/v1/query?query=count(up%20%3D%3D%201)" || exit 1
|
|
n=$(grep -oE "\"[0-9]+\"\\]" /tmp/helper_body | head -1 | grep -oE "[0-9]+")
|
|
echo "n=$n"
|
|
test -n "$n" && test "$n" -ge 25
|
|
'
|
|
|
|
# Alertmanager: nothing critical firing right now. Empty array == clean.
|
|
check "alertmanager: no severity=critical alerts active" bash -c '
|
|
helper_curl "http://kube-prometheus-stack-alertmanager.monitoring.svc:9093/api/v2/alerts?filter=severity%3Dcritical&active=true" || exit 1
|
|
body=$(cat /tmp/helper_body)
|
|
echo "body=$body"
|
|
test "$body" = "[]"
|
|
'
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Local AI inference. Every check here corresponds to a defect that actually
|
|
# shipped, because none of them are catchable by manifest validation — the
|
|
# manifests were valid in each case and the endpoint still misbehaved.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
export LB_IP
|
|
|
|
# helper_post: POST JSON from inside e2e-helper, for Services with no ingress.
|
|
# kubectl exec passes argv directly, so the JSON needs no shell quoting.
|
|
helper_post() {
|
|
local url="$1" data="$2" out body code attempt
|
|
for attempt in 1 2 3; do
|
|
out=$(kubectl -n monitoring exec e2e-helper -- \
|
|
curl -sk --max-time 60 -H 'Content-Type: application/json' \
|
|
-d "$data" -w $'\n__CODE__:%{http_code}' "$url" 2>&1)
|
|
body=${out%$'\n'__CODE__:*}
|
|
code=${out##*__CODE__:}
|
|
printf 'attempt=%s code=%s body=%s\n' "$attempt" "$code" "${body:0:200}"
|
|
if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
|
|
printf '%s' "$body" > /tmp/helper_body
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
export -f helper_post
|
|
|
|
# llm_chat: POST a chat completion to llama-server from inside the cluster.
|
|
# It goes to the Service, not the public hostname: llm.renner.uno now answers
|
|
# with an Authentik redirect, and the API is deliberately in-cluster only —
|
|
# which is also how every real consumer reaches it. Result lands in
|
|
# /tmp/helper_body.
|
|
llm_chat() {
|
|
local prompt="$1" max="${2:-64}"
|
|
helper_post "http://llama-server.ai.svc.cluster.local:8080/v1/chat/completions" \
|
|
"$(jq -nc --arg p "$prompt" --argjson n "$max" \
|
|
'{model:"qwen3.6-35b-a3b",messages:[{role:"user",content:$p}],max_tokens:$n,temperature:0}')"
|
|
}
|
|
export -f llm_chat
|
|
|
|
echo "=== Local AI inference ==="
|
|
|
|
# The gate is the whole of the access control on the public hostname:
|
|
# llama-server has no authentication and serves the API on the same port as the
|
|
# UI. An anonymous request must be bounced to Authentik, never answered. A 200
|
|
# here would mean the ingress is reaching llama-server directly again.
|
|
check "public hostname is gated by authentik" \
|
|
bash -c 'code=$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
|
|
-o /dev/null -w "%{http_code}" "https://llm.renner.uno/v1/models")
|
|
echo "got=$code"; [[ "$code" =~ ^(302|307|401|403)$ ]]'
|
|
|
|
# Asserts the id specifically, not merely that the string appears somewhere:
|
|
# every configured name shows up in the `aliases` array regardless of order, so
|
|
# a substring match passes even when the picker is showing the wrong one.
|
|
check "descriptive name is the advertised id" \
|
|
bash -c 'helper_curl "http://llama-server.ai.svc.cluster.local:8080/v1/models" &&
|
|
jq -e ".data[0].id == \"qwen3.6-35b-a3b\"" /tmp/helper_body >/dev/null'
|
|
|
|
# Only one name is configured. A second would be advertised alongside it and,
|
|
# if it sorted first, would take the id — which is what a picker displays.
|
|
check "no second name competes for the id" \
|
|
bash -c 'jq -e ".data[0].aliases | length == 1" /tmp/helper_body >/dev/null'
|
|
|
|
# Regression: the embedding model used to share this port. It appeared in the
|
|
# chat picker, and selecting it returned fluent nonsense ("capital capital
|
|
# capital ...") instead of an error, because --embeddings does not refuse chat
|
|
# completions. It now lives on a separate Service; if it ever reappears here,
|
|
# it is selectable again and this must fail.
|
|
check "embedding model absent from chat endpoint" \
|
|
bash -c '! grep -qi embed /tmp/helper_body'
|
|
|
|
# Regression: with thinking enabled by default the model spent its whole token
|
|
# budget on reasoning_content and returned an EMPTY content with
|
|
# finish_reason "length" — a 200 response that answers nothing. Asserting on
|
|
# the status code alone would have passed.
|
|
check "chat completion returns non-empty content" \
|
|
bash -c 'llm_chat "Nenne die Hauptstadt von Frankreich." 64 &&
|
|
jq -e ".choices[0].message.content | length > 0" /tmp/helper_body >/dev/null'
|
|
|
|
check "chat completion stops rather than truncating" \
|
|
bash -c 'jq -e ".choices[0].finish_reason == \"stop\"" /tmp/helper_body >/dev/null'
|
|
|
|
# Coherence, not just liveness. This is what catches a model answering from the
|
|
# wrong weights or a numerically broken backend — llama-bench measures
|
|
# throughput and would report excellent numbers while emitting garbage.
|
|
check "chat answer is coherent (knows Paris)" \
|
|
bash -c 'jq -re ".choices[0].message.content" /tmp/helper_body | grep -qi paris'
|
|
|
|
# Reached from the monitoring namespace, which ai-ns.yaml admits; there is no
|
|
# ingress for this Service by design.
|
|
check "llama-embed returns an embedding vector" \
|
|
bash -c 'helper_post "http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings" \
|
|
"{\"model\":\"embed\",\"input\":\"Testsatz\"}" &&
|
|
jq -e ".data[0].embedding | length > 0" /tmp/helper_body >/dev/null'
|
|
|
|
# Guards the resource request silently disappearing: without it the pod still
|
|
# runs, just on CPU, and only a throughput measurement would notice.
|
|
check "llama-server holds an iGPU allocation" \
|
|
bash -c 'kubectl -n ai get deploy llama-server -o json |
|
|
jq -e ".spec.template.spec.containers[0].resources.limits[\"gpu.intel.com/i915\"]" >/dev/null'
|
|
|
|
echo "=== Authentik OIDC discovery (per-app) ==="
|
|
# A reachable, parseable .well-known proves Authentik has an OIDC provider
|
|
# bound to that application slug. If the app config is missing or Authentik
|
|
# hasn't reconciled, this is a 404 / non-JSON.
|
|
oidc_check() {
|
|
local slug="$1" body
|
|
body=$(curl -sk --resolve "auth.renner.uno:443:$LB_IP" --max-time 10 \
|
|
"https://auth.renner.uno/application/o/$slug/.well-known/openid-configuration")
|
|
echo "$body" | head -c 200
|
|
echo "$body" | grep -qE "\"issuer\".*auth\\.renner\\.uno" && \
|
|
echo "$body" | grep -q "\"authorization_endpoint\""
|
|
}
|
|
for slug in audiobookshelf grafana; do
|
|
check "authentik OIDC well-known: $slug" oidc_check "$slug"
|
|
done
|
|
|
|
# Tear down the helper pod regardless of pass/fail so reruns are clean.
|
|
kubectl -n monitoring delete pod e2e-helper --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo " Passed: $PASS"
|
|
echo " Failed: $FAIL"
|
|
echo "========================================"
|
|
if [[ "$FAIL" -gt 0 ]]; then
|
|
printf ' * %s\n' "${FAILURES[@]}"
|
|
exit 1
|
|
fi
|
|
echo "All checks passed."
|