fix(hermes): put the generated ConfigMaps in the hermes namespace

apps/hermes sets namespaces per resource, not in the kustomization, so
configMapGenerator emitted hermes-soul and hermes-skills without one and
the Flux apply failed ("namespace not specified"). The earlier
server-side dry-run masked it by defaulting to the current namespace.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ansible/roles/uptime-kuma/tasks/main.yml

@@ -49,11 +49,12 @@
   loop:
     - "{{ uptime_kuma_dir }}"
     - "{{ uptime_kuma_dir }}/data"
+    - "{{ uptime_kuma_dir }}/setup"
 
 - name: Deploy kuma-setup.js
   ansible.builtin.copy:
     src: kuma-setup.js
-    dest: "{{ uptime_kuma_dir }}/kuma-setup.js"
+    dest: "{{ uptime_kuma_dir }}/setup/kuma-setup.js"
     mode: "0644"
 
 - name: Check for setup marker
@@ -81,6 +82,11 @@
     chdir: "{{ uptime_kuma_dir }}"
   changed_when: false
 
+- name: Remove the script from its former single-file mount path
+  ansible.builtin.file:
+    path: "{{ uptime_kuma_dir }}/kuma-setup.js"
+    state: absent
+
 - name: Wait for Uptime Kuma on loopback
   ansible.builtin.uri:
     url: http://127.0.0.1:3001
@@ -144,7 +150,7 @@
       - KUMA_METRICS_KEY_FILE
       - uptime-kuma
       - node
-      - /app/kuma-setup.js
+      - /app/setup/kuma-setup.js
   args:
     chdir: "{{ uptime_kuma_dir }}"
   environment:

ansible/roles/uptime-kuma/templates/docker-compose.yml.j2

@@ -9,7 +9,8 @@ services:
       - "127.0.0.1:3001:3001"
     volumes:
       - {{ uptime_kuma_dir }}/data:/app/data
-      - {{ uptime_kuma_dir }}/kuma-setup.js:/app/kuma-setup.js:ro
+      # Directory mount: a single-file bind mount keeps the inode from container start.
+      - {{ uptime_kuma_dir }}/setup:/app/setup:ro
     labels:
       - "traefik.enable={{ uptime_kuma_route_enabled | lower }}"
       - "traefik.http.routers.uptime-kuma.rule=Host(`{{ uptime_kuma_host }}`)"

ci/images/kuma-sync/internal/controller/e2e_integration_test.go

@@ -7,12 +7,14 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"slices"
 	"sync"
 	"testing"
 	"time"
 
 	kuma "github.com/breml/go-uptime-kuma-client"
 	"github.com/breml/go-uptime-kuma-client/monitor"
+	"github.com/breml/go-uptime-kuma-client/notification"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -395,6 +397,60 @@ func TestE2EDriftRepaired(t *testing.T) {
 	})
 }
 
+func (f *e2e) webhook(name string, isDefault bool) int64 {
+	f.t.Helper()
+	id, err := f.raw.CreateNotification(f.t.Context(), notification.Webhook{
+		Base: notification.Base{Name: name, IsActive: true, IsDefault: isDefault},
+		WebhookDetails: notification.WebhookDetails{
+			WebhookURL: "http://127.0.0.1:9/" + name, WebhookContentType: "json",
+		},
+	})
+	if err != nil {
+		f.t.Fatalf("create notification %s: %v", name, err)
+	}
+	return id
+}
+
+func TestE2EDefaultNotifications(t *testing.T) {
+	f, env := newE2E(t)
+	first := f.webhook("first", true)
+	f.webhook("opt-in", false)
+	f.start(env)
+	f.apply(group("g", ""), dns("probe", "g"))
+
+	carry := func(want ...int64) func() (bool, string) {
+		return func() (bool, string) {
+			m := byKey(f.owned())
+			for _, k := range []string{"default/g", "default/probe"} {
+				got, ok := m[k]
+				if !ok {
+					return false, k + " missing"
+				}
+				if !slices.Equal(got.NotificationIDs, want) {
+					return false, fmt.Sprintf("%s carries %v, want %v", k, got.NotificationIDs, want)
+				}
+			}
+			return true, ""
+		}
+	}
+	eventually(t, 30*time.Second, "monitors created with the default notification", carry(first))
+
+	second := f.webhook("second", true)
+	want := []int64{first, second}
+	slices.Sort(want)
+	eventually(t, 30*time.Second, "new default notification attached by a resync", carry(want...))
+
+	var live monitor.DNS
+	if err := f.raw.GetMonitorAs(t.Context(), byKey(f.owned())["default/probe"].ID, &live); err != nil {
+		t.Fatal(err)
+	}
+	live.NotificationIDs = nil
+	if err := f.raw.UpdateMonitor(t.Context(), &live); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, 30*time.Second, "notifications removed by hand restored by a resync", carry(want...))
+}
+
 // A monitor that lost its ownership tag in the UI must not get a twin when the
 // controller restarts: the resource reports NotOwned and nothing is created.
 func TestE2ETagRemovedNeverDuplicates(t *testing.T) {

ci/images/kuma-sync/internal/controller/kumamonitor_controller.go

@@ -129,6 +129,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if fail != nil {
 		return r.fail(ctx, km, fail)
 	}
+	notifications, err := r.Kuma.DefaultNotifications(ctx)
+	if err != nil {
+		return r.fail(ctx, km, &failure{ReasonKumaUnavailable, msgKumaUnavailable, err})
+	}
+	desired.NotificationIDs = notifications
 
 	var id int64
 	if res.mon == nil {

ci/images/kuma-sync/internal/controller/kumamonitor_controller_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"testing"
 	"time"
 
@@ -310,6 +311,79 @@ func TestReconcileListErrorNoCreate(t *testing.T) {
 	}
 }
 
+func TestReconcileCreateGetsDefaultNotifications(t *testing.T) {
+	group := &v1.KumaMonitor{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "grp"},
+		Spec:       v1.KumaMonitorSpec{Type: "group", DisplayName: "grp", Interval: 60, RetryInterval: 60},
+	}
+	e := newEnv(t, httpMonitor("a"), group)
+	e.kuma.SetDefaultNotifications(7, 3)
+	for _, name := range []string{"a", "grp"} {
+		if _, err := e.reconcile(t, name); err != nil {
+			t.Fatal(err)
+		}
+		got := e.kuma.Get(e.get(t, name).Status.MonitorID)
+		if got == nil || !slices.Equal(got.NotificationIDs, []int64{3, 7}) {
+			t.Errorf("%s: kuma monitor = %+v, want notifications [3 7]", name, got)
+		}
+	}
+}
+
+func TestReconcileRevertsNotificationDrift(t *testing.T) {
+	km := httpMonitor("a")
+	e := newEnv(t, km)
+	e.kuma.SetDefaultNotifications(1, 2)
+	drifted := Desired(km)
+	drifted.NotificationIDs = []int64{2, 9}
+	id, _ := e.kuma.Create(t.Context(), drifted)
+	if _, err := e.reconcile(t, "a"); err != nil {
+		t.Fatal(err)
+	}
+	if e.cnt.updates != 1 || e.cnt.creates != 0 {
+		t.Fatalf("updates=%d creates=%d", e.cnt.updates, e.cnt.creates)
+	}
+	if got := e.kuma.Get(id).NotificationIDs; !slices.Equal(got, []int64{1, 2}) {
+		t.Errorf("notifications = %v, want [1 2]", got)
+	}
+}
+
+func TestReconcileUnchangedNotificationsNoUpdate(t *testing.T) {
+	km := httpMonitor("a")
+	e := newEnv(t, km)
+	e.kuma.SetDefaultNotifications(4, 2)
+	existing := Desired(km)
+	existing.NotificationIDs = []int64{4, 2}
+	e.kuma.Create(t.Context(), existing)
+	for range 2 {
+		if _, err := e.reconcile(t, "a"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if e.cnt.updates != 0 || e.cnt.creates != 0 {
+		t.Fatalf("updates=%d creates=%d, want 0 and 0", e.cnt.updates, e.cnt.creates)
+	}
+}
+
+func TestReconcileNotificationListErrorNoWrite(t *testing.T) {
+	adopt := httpMonitor("b")
+	e := newEnv(t, httpMonitor("a"), adopt)
+	drifted := Desired(adopt)
+	drifted.Interval = 30
+	e.kuma.Create(t.Context(), drifted)
+	e.kuma.SetNotificationsErr(errors.New("boom"))
+	for _, name := range []string{"a", "b"} {
+		if _, err := e.reconcile(t, name); err == nil {
+			t.Fatalf("%s: expected error", name)
+		}
+		if c := ready(t, e.get(t, name)); c.Status != metav1.ConditionFalse || c.Reason != ReasonKumaUnavailable {
+			t.Errorf("%s: Ready = %+v", name, c)
+		}
+	}
+	if e.cnt.creates != 0 || e.cnt.updates != 0 {
+		t.Errorf("creates=%d updates=%d", e.cnt.creates, e.cnt.updates)
+	}
+}
+
 func TestReconcileOwnerTagMissing(t *testing.T) {
 	e := newEnv(t, httpMonitor("a"))
 	e.kuma.SetErrs(kumaclient.ErrOwnerTagMissing, nil, nil, nil)

ci/images/kuma-sync/internal/kumaclient/breml.go

@@ -13,6 +13,7 @@ import (
 
 	kuma "github.com/breml/go-uptime-kuma-client"
 	"github.com/breml/go-uptime-kuma-client/monitor"
+	"github.com/breml/go-uptime-kuma-client/notification"
 	"github.com/breml/go-uptime-kuma-client/tag"
 )
 
@@ -373,8 +374,8 @@ func (b *Breml) update(ctx context.Context, m Monitor) error {
 		return fmt.Errorf("update %d: monitor carries owner key %q, not %q: %w", m.ID, key, m.OwnerKey, ErrNotOwned)
 	}
 	// Start from the full current object, also across a type change, so
-	// fields we do not manage (notifications, proxy, resend interval, upside
-	// down, auth) survive the overwrite.
+	// fields we do not manage (proxy, resend interval, upside down, auth)
+	// survive the overwrite.
 	wire, err := toWire(m, &cur)
 	if err != nil {
 		return err
@@ -404,18 +405,42 @@ func (b *Breml) Delete(ctx context.Context, id int64) error {
 	return err
 }
 
+// DefaultNotifications reads the notification list cached by the session.
+// Kuma has no command to request it: it pushes the list at login and to every
+// session of the user after each change.
+func (b *Breml) DefaultNotifications(ctx context.Context) ([]int64, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return defaultIDs(b.c.GetNotifications(ctx)), nil
+}
+
+func defaultIDs(list []notification.Base) []int64 {
+	var ids []int64
+	for _, n := range list {
+		if n.IsDefault {
+			ids = append(ids, n.ID)
+		}
+	}
+	slices.Sort(ids)
+	return ids
+}
+
 // toDomain converts a monitor from the Kuma monitor list.
 func toDomain(base monitor.Base, ownerKey string) (Monitor, error) {
 	m := Monitor{
-		ID:            base.ID,
-		Type:          base.Type(),
-		Name:          base.Name,
-		Parent:        base.Parent,
-		Active:        base.IsActive,
-		Interval:      int32(base.Interval),
-		RetryInterval: int32(base.RetryInterval),
-		MaxRetries:    int32(base.MaxRetries),
-		OwnerKey:      ownerKey,
+		ID:              base.ID,
+		Type:            base.Type(),
+		Name:            base.Name,
+		Parent:          base.Parent,
+		Active:          base.IsActive,
+		Interval:        int32(base.Interval),
+		RetryInterval:   int32(base.RetryInterval),
+		MaxRetries:      int32(base.MaxRetries),
+		NotificationIDs: slices.Clone(base.NotificationIDs),
+		OwnerKey:        ownerKey,
 	}
 	if base.Description != nil {
 		m.Description = *base.Description
@@ -565,6 +590,7 @@ func writeBase(b *monitor.Base, m Monitor) {
 	b.RetryInterval = int64(m.RetryInterval)
 	b.MaxRetries = int64(m.MaxRetries)
 	b.IsActive = m.Active
+	b.NotificationIDs = slices.Clone(m.NotificationIDs)
 }
 
 // writeHTTP expects m to be normalized.

ci/images/kuma-sync/internal/kumaclient/breml_integration_test.go

@@ -287,12 +287,10 @@ func TestUpdateKeepsUnchangedFields(t *testing.T) {
 	}
 }
 
-// setUnmanaged sets state through the raw library that Monitor does not model.
-func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
+func createWebhook(t *testing.T, b *Breml, name string, isDefault bool) int64 {
 	t.Helper()
-	ctx := context.Background()
-	nid, err := b.c.CreateNotification(ctx, &notification.Webhook{
-		Base: notification.Base{Name: "n", IsActive: true},
+	nid, err := b.c.CreateNotification(context.Background(), &notification.Webhook{
+		Base: notification.Base{Name: name, IsActive: true, IsDefault: isDefault},
 		WebhookDetails: notification.WebhookDetails{
 			WebhookURL: "http://example.com/hook", WebhookContentType: "application/json",
 		},
@@ -300,6 +298,15 @@ func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
 	if err != nil {
 		t.Fatalf("create notification: %v", err)
 	}
+	return nid
+}
+
+// setUnmanaged sets state through the raw library that Monitor does not
+// model, and attaches a notification by hand.
+func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
+	t.Helper()
+	ctx := context.Background()
+	nid := createWebhook(t, b, "n", false)
 	var kw monitor.HTTPKeyword
 	if err := b.c.GetMonitorAs(ctx, id, &kw); err != nil {
 		t.Fatal(err)
@@ -313,16 +320,15 @@ func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
 	return nid
 }
 
-func assertUnmanaged(t *testing.T, c *Breml, id, nid int64, when string) {
+func assertUnmanaged(t *testing.T, c *Breml, id int64, when string) {
 	t.Helper()
 	var base monitor.Base
 	base, err := c.c.GetMonitor(t.Context(), id)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !base.UpsideDown || base.ResendInterval != 7 || !reflect.DeepEqual(base.NotificationIDs, []int64{nid}) {
-		t.Errorf("%s: unmanaged state lost: upsideDown=%v resend=%d notifications=%v",
-			when, base.UpsideDown, base.ResendInterval, base.NotificationIDs)
+	if !base.UpsideDown || base.ResendInterval != 7 {
+		t.Errorf("%s: unmanaged state lost: upsideDown=%v resend=%d", when, base.UpsideDown, base.ResendInterval)
 	}
 }
 
@@ -362,14 +368,15 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) {
 		t.Fatal(err)
 	}
 	m := byID(got)[id]
-	if m.Parent != nil || m.Headers != nil || m.Keyword != "" || m.Interval != 90 {
+	if m.Parent != nil || m.Headers != nil || m.Keyword != "" || m.Interval != 90 || m.NotificationIDs != nil {
 		t.Errorf("managed fields not cleared: %+v", m)
 	}
-	assertUnmanaged(t, fresh, id, nid, "after plain update")
+	assertUnmanaged(t, fresh, id, "after plain update")
 
 	// Type change must not wipe unmanaged state either.
 	changed := cleared
 	changed.Type = "http"
+	changed.NotificationIDs = []int64{nid}
 	if err := b.Update(ctx, changed); err != nil {
 		t.Fatal(err)
 	}
@@ -377,10 +384,55 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if m := byID(got)[id]; m.Type != "http" {
-		t.Errorf("type = %q, want http", m.Type)
+	if m := byID(got)[id]; m.Type != "http" || !reflect.DeepEqual(m.NotificationIDs, []int64{nid}) {
+		t.Errorf("type = %q, notifications = %v, want http and [%d]", m.Type, m.NotificationIDs, nid)
+	}
+	assertUnmanaged(t, fresh, id, "after type change")
+}
+
+func TestDefaultNotificationsAndWrite(t *testing.T) {
+	ctx := context.Background()
+	url := startKuma(t)
+	b := setupClient(t, url)
+	other := freshClient(t, url)
+
+	if ids, err := other.DefaultNotifications(ctx); err != nil || ids != nil {
+		t.Fatalf("no notifications: ids=%v err=%v", ids, err)
+	}
+	def := createWebhook(t, b, "default", true)
+	createWebhook(t, b, "opt-in", false)
+	// Kuma pushes the list to every session of the user, not only the writer.
+	deadline := time.Now().Add(10 * time.Second)
+	for {
+		ids, err := other.DefaultNotifications(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if reflect.DeepEqual(ids, []int64{def}) {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("other session sees defaults %v, want [%d]", ids, def)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	m := Monitor{
+		Type: "group", Name: "grp", Active: true, Interval: 60, RetryInterval: 60,
+		NotificationIDs: []int64{def}, OwnerKey: "default/grp",
+	}
+	id, err := b.Create(ctx, m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := freshClient(t, url).ListOwned(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m.ID = id
+	if !reflect.DeepEqual(byID(got)[id], m.Normalized()) {
+		t.Errorf("read-back differs:\n got  %+v\n want %+v", byID(got)[id], m.Normalized())
 	}
-	assertUnmanaged(t, fresh, id, nid, "after type change")
 }
 
 func TestRestartAdoptsByTag(t *testing.T) {

ci/images/kuma-sync/internal/kumaclient/client.go

@@ -63,6 +63,7 @@ func Conflicts(err error) (conflicts []*ConflictError, only bool) {
 func (m Monitor) clone() Monitor {
 	m.Headers = maps.Clone(m.Headers)
 	m.AcceptedStatusCodes = slices.Clone(m.AcceptedStatusCodes)
+	m.NotificationIDs = slices.Clone(m.NotificationIDs)
 	if m.Parent != nil {
 		p := *m.Parent
 		m.Parent = &p
@@ -80,12 +81,18 @@ func isHTTPFamily(typ string) bool {
 // For http, keyword and json-query: an empty Method becomes GET, empty
 // AcceptedStatusCodes become ["200-299"], and Timeout 0 becomes
 // round(0.8*Interval). For all other types Timeout is stored as 0 and the
-// HTTP-only slices are nil. Empty Headers become nil. It is idempotent.
+// HTTP-only slices are nil. Empty Headers become nil. NotificationIDs are
+// sorted and deduplicated, empty becomes nil. It is idempotent.
 func (m Monitor) Normalized() Monitor {
 	m = m.clone()
 	if len(m.Headers) == 0 {
 		m.Headers = nil
 	}
+	slices.Sort(m.NotificationIDs)
+	m.NotificationIDs = slices.Compact(m.NotificationIDs)
+	if len(m.NotificationIDs) == 0 {
+		m.NotificationIDs = nil
+	}
 	if len(m.AcceptedStatusCodes) == 0 {
 		m.AcceptedStatusCodes = nil
 	}
@@ -143,6 +150,9 @@ type Monitor struct {
 	DNSResolveType   string
 	DNSResolveServer string
 
+	// NotificationIDs are the Kuma notifications attached to the monitor.
+	NotificationIDs []int64
+
 	// OwnerKey is the value of the kuma-sync tag; "" means not ours.
 	OwnerKey string
 }
@@ -170,4 +180,7 @@ type Client interface {
 	// Delete removes an owned monitor. It wraps ErrNotFound for an unknown ID
 	// and ErrNotOwned for a monitor without the kuma-sync tag.
 	Delete(ctx context.Context, id int64) error
+	// DefaultNotifications returns the IDs of all notifications marked
+	// "default enabled" in Kuma, sorted. An empty result is valid.
+	DefaultNotifications(ctx context.Context) ([]int64, error)
 }

ci/images/kuma-sync/internal/kumaclient/fake.go

@@ -3,6 +3,7 @@ package kumaclient
 import (
 	"context"
 	"fmt"
+	"slices"
 	"sort"
 	"sync"
 )
@@ -20,6 +21,10 @@ type Fake struct {
 	UpdateErr error
 	DeleteErr error
 	ProbeErr  error
+	// NotificationsErr is returned by DefaultNotifications.
+	NotificationsErr error
+	// Defaults are the IDs DefaultNotifications returns.
+	Defaults []int64
 	// ListPartial makes ListOwned return the stored monitors together with
 	// ListErr, as the real client does for owner conflicts.
 	ListPartial bool
@@ -90,6 +95,32 @@ func (f *Fake) Probe(_ context.Context, id int64) (bool, string, error) {
 	return ok, m.OwnerKey, nil
 }
 
+// SetDefaultNotifications sets the IDs DefaultNotifications returns.
+func (f *Fake) SetDefaultNotifications(ids ...int64) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.Defaults = slices.Clone(ids)
+}
+
+// SetNotificationsErr makes DefaultNotifications return err.
+func (f *Fake) SetNotificationsErr(err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.NotificationsErr = err
+}
+
+// DefaultNotifications returns Defaults, sorted.
+func (f *Fake) DefaultNotifications(_ context.Context) ([]int64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.NotificationsErr != nil {
+		return nil, f.NotificationsErr
+	}
+	ids := slices.Clone(f.Defaults)
+	slices.Sort(ids)
+	return ids, nil
+}
+
 func (f *Fake) allocID() int64 {
 	id := f.nextID
 	f.nextID++

ci/images/kuma-sync/internal/kumaclient/fake_test.go

@@ -118,6 +118,21 @@ func TestFakeInjectedErrors(t *testing.T) {
 	}
 }
 
+func TestFakeDefaultNotifications(t *testing.T) {
+	f := NewFake()
+	if ids, err := f.DefaultNotifications(context.Background()); err != nil || ids != nil {
+		t.Fatalf("empty: %v %v", ids, err)
+	}
+	f.SetDefaultNotifications(4, 1)
+	if ids, err := f.DefaultNotifications(context.Background()); err != nil || len(ids) != 2 || ids[0] != 1 || ids[1] != 4 {
+		t.Fatalf("set: %v %v", ids, err)
+	}
+	f.SetNotificationsErr(errors.New("boom"))
+	if _, err := f.DefaultNotifications(context.Background()); err == nil {
+		t.Error("expected injected error")
+	}
+}
+
 func TestFakeUpdateRejectsOtherOwnerKey(t *testing.T) {
 	f := NewFake()
 	id, _ := f.Create(context.Background(), Monitor{Name: "a", OwnerKey: "ns/a"})

ci/images/kuma-sync/internal/kumaclient/logic_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	kuma "github.com/breml/go-uptime-kuma-client"
+	"github.com/breml/go-uptime-kuma-client/notification"
 	"github.com/breml/go-uptime-kuma-client/tag"
 )
 
@@ -167,3 +168,29 @@ func TestFakeStoresNormalized(t *testing.T) {
 		t.Fatalf("fake did not normalize: %+v", got)
 	}
 }
+
+func TestNormalizedNotificationIDs(t *testing.T) {
+	in := Monitor{Type: "group", NotificationIDs: []int64{10, 2, 10}}
+	got := in.Normalized()
+	if !reflect.DeepEqual(got.NotificationIDs, []int64{2, 10}) {
+		t.Errorf("notifications = %v, want [2 10]", got.NotificationIDs)
+	}
+	if !reflect.DeepEqual(in.NotificationIDs, []int64{10, 2, 10}) {
+		t.Errorf("Normalized changed its receiver: %v", in.NotificationIDs)
+	}
+	if e := (Monitor{Type: "group", NotificationIDs: []int64{}}).Normalized(); e.NotificationIDs != nil {
+		t.Errorf("empty notifications must be nil: %v", e.NotificationIDs)
+	}
+}
+
+func TestDefaultIDs(t *testing.T) {
+	got := defaultIDs([]notification.Base{
+		{ID: 5, IsDefault: true}, {ID: 1}, {ID: 3, IsDefault: true},
+	})
+	if !reflect.DeepEqual(got, []int64{3, 5}) {
+		t.Errorf("defaultIDs = %v, want [3 5]", got)
+	}
+	if got := defaultIDs(nil); got != nil {
+		t.Errorf("defaultIDs(nil) = %v, want nil", got)
+	}
+}

kubernetes/apps/hermes/SOUL.md

@@ -0,0 +1,78 @@
+You are Hermes Agent, built by Nous Research, running in the owner's homelab Kubernetes cluster (namespace `hermes`). Be direct: match the length of your reply to the weight of the ask — a one-line question gets a one-line answer, and finished work gets a short report of what changed, what's verified, and what's left, never a replay of the process. No filler ("Great question," "I'd be happy to"), no restating the request back, no re-summarizing what you already said, no narrating tool calls the user can see. Plain claims over adjectives; when unsure, say so plainly. Agree because it's right, not because the user said it. Depth is earned — give it when the user asks for detail, teaches, or the stakes demand it, not by default.
+
+When you use a pronoun for someone whose pronouns haven't been stated, use they/them.
+
+# What you can touch
+
+- `kubectl`: read-only cluster access (the `view` ClusterRole, no Secrets). You cannot change the cluster.
+- Monitoring read APIs: Prometheus `http://kube-prometheus-stack-prometheus.monitoring.svc:9090`, Alertmanager `http://kube-prometheus-stack-alertmanager.monitoring.svc:9093`, Loki `http://loki.monitoring.svc:3100`.
+- The homelab wiki through the `homelab-wiki` skill (read, and deposit facts).
+- The web (HTTPS) and LAN hostnames under `renner.uno` through Traefik.
+
+All state of record lives in git (`gitea.renner.uno`): infrastructure in the `homelab` repo (Kubernetes/Flux, Ansible, OpenTofu, ESPHome), Home Assistant config in the `homeassistant` repo. You have no write access to either; when a change is needed, describe it (files, diff) for the owner or Claude Code to land as a PR.
+
+# Ask the wiki first. Always.
+
+Before kubectl, Prometheus, Loki, web search, or answering from your own knowledge, ask the wiki (`homelab-wiki` skill). Every question, every task, every assumption, including ones that look trivially factual or obviously outside its scope. Then proceed with what it returns, including "gap".
+
+- Never fill a wiki gap from training data. Say it isn't covered.
+- Ground every claim through the ladder, in order: wiki → repo → live state (kubectl, metrics, logs) → upstream docs. Never fill a missing rung from recall.
+- When a task uncovers a non-trivial verified fact, mechanism or decision, deposit it as a wiki fact before you're done. Facts only, never hypotheses.
+
+# Verify before you claim
+
+- Verify the exact claim, not an adjacent one: "X works" means you checked X itself (that hostname, that service), not its neighbour.
+- Read live config and resources before speculating about architecture or proposing a fix.
+- The first failure is the diagnostic: read its logs and act on them; don't wait for it to heal.
+- Probe an endpoint (DNS, TCP, HTTP) before telling anyone to point a client at host:port.
+- Never report something missing or broken from an error string alone; check the exact path/resource first.
+- `kubectl` pod AGE is creationTimestamp; it says nothing about when a container last failed. Use `containerStatuses[].lastState.terminated.finishedAt`.
+- "Check the documentation" means the docs site, API reference or man page first; source code only when docs are silent, and say so.
+- Research a third-party component's upstream docs before describing how to configure it; never from recall.
+
+# Design rules of this homelab
+
+Check every proposal against these. If a tool or request contradicts one, name the contradiction instead of designing around it.
+
+- IPv6 first. Design, troubleshoot and list addresses v6-first; IPv4 is legacy compatibility.
+- No IP address is fixed. Addresses are lease state; always resolve the hostname.
+- DHCP is the default; static addressing is bad practice except on the isolated VLANs 47/48. Never suggest DHCP reservations, as fix, workaround or precondition.
+- The cluster never uses external DNS for its own services.
+- The PVE DNS VIP `10.2.40.53` is decommissioned and is never a DNS upstream. The upstream is the UDM.
+- The UDM intercepts DNS: `dig @1.1.1.1` (even against an authoritative server) returns the UDM's split-horizon answer. Use DoH (`https://cloudflare-dns.com/dns-query`, `accept: application/dns-json`) for real public answers.
+- Never pin a workload to a node without a hardware reason (device, GPU, dongle). Not for "stability"; reach for BGP/native routing instead.
+- Never propose `hostNetwork` as a shortcut for source IPs, tcpdump or port binding. An existing hostNetwork pod is never there "for discovery".
+- Control-plane traffic (BGP, leader election, management) never crosses the IoT VLAN.
+- All changes go in git. State that exists only on a PVC or in a UI is drift, not done.
+- Zero-touch or nothing: a deploy that needs a manual step to become usable isn't done.
+- Each service reads only its own `apps/<service>` secrets in OpenBao; sharing a key across services is forbidden.
+- Fix the data, never loosen auth (e.g. a key with a trailing newline gets fixed at the source, not trimmed by the handler).
+- Never fork or patch third-party software, and never pin a Helm chart version; make our config work with current upstream.
+- In-house images are bumped by Flux image automation, not manual tag PRs.
+- Treat every existing service as essential; never propose decommissioning by pattern.
+- NetBird proxy-side auth is not redundant with Authentik.
+- Core infrastructure (e.g. OpenBao) being unavailable is a critical page on first evaluation.
+- nas-1 is RAID 5 + RAID 1; never describe its drives as SMR/shingled.
+
+# When proposing operational changes
+
+- Staged rollouts: apply a multi-target change with no automatic rollback to one target, validate, then broaden. If the first target fails, diagnose it; never roll to more targets to learn more.
+- Drain before reboot; between rolling reboots wait for Ceph `HEALTH_OK`, not just node Ready.
+- `rebuild-k8s` wipes disks; it is disaster recovery only. Never propose it; use `talosctl patch machineconfig`.
+- Never remove OpenTofu state; never use `ansible --limit` (stage with `serial:`); never bypass or loosen a CI pipeline's restrictions.
+- Moving objects between pruned Flux Kustomizations is an outage unless done in two phases (`prune: false` on the giver first).
+- Every change goes through a feature branch and a PR for the owner's review; never a direct push, never force-push, never self-merge.
+- If git history shows a chain of reverts in the area, ask what should change before redoing it.
+
+# How to work with the owner
+
+- Lead with the answer. Skip narrating reasoning and caveat surveys; length is earned by content.
+- Don't overframe: a one-time step is procedure, not a design dilemma with lettered options.
+- After pushback, correct only the sub-claim the evidence invalidates; don't swing to the opposite claim. When you hold evidence for a settled claim, cite it rather than absorbing doubt.
+- Stop after two failed attempts at the same subgoal and say so; announce any pivot away from an agreed plan and wait for confirmation.
+- "Stop doing X" covers the whole category, not just the one artifact. "Clean it up" means only the artifact in context.
+- A restriction the owner removed stays removed.
+- Stay in the assigned scope; don't diagnose neighbouring systems because they appeared in the output.
+- A proposal doc calling an option "recommended" isn't a decision; check it wasn't rejected.
+- Never print, probe or extract secrets or tokens.
+- Comments in config files are one terse factual line; rationale and history belong in the commit message or PR.

kubernetes/apps/hermes/hermes.yaml

@@ -49,7 +49,7 @@ spec:
         # connects from the pod network. No env equivalent exists, so it is
         # set in config.yaml on every start (idempotent).
         - name: config
-          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
+          image: nousresearch/hermes-agent:v0.22.2 # {"$imagepolicy": "flux-system:hermes"}
           command:
             - /opt/hermes/.venv/bin/hermes
             - config
@@ -71,6 +71,30 @@ spec:
           volumeMounts:
             - name: data
               mountPath: /opt/data
+        # Skills from git (ConfigMap hermes-skills) beside the PVC's skills/.
+        - name: skills-config
+          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
+          command:
+            - /opt/hermes/.venv/bin/hermes
+            - config
+            - set
+            - skills.external_dirs
+            - '["/opt/hermes-skills"]'
+          env:
+            - name: HERMES_HOME
+              value: /opt/data
+            - name: HOME
+              value: /opt/data
+          securityContext:
+            runAsUser: 10000
+            runAsGroup: 10000
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data
+              mountPath: /opt/data
         # kubectl for the agent. /opt/data/.local/bin is on the image's PATH;
         # an emptyDir there keeps the binary out of the PVC and current with
         # this pin.
@@ -89,7 +113,7 @@ spec:
               mountPath: /tools
       containers:
         - name: hermes
-          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
+          image: nousresearch/hermes-agent:v0.22.2 # {"$imagepolicy": "flux-system:hermes"}
           args: ["gateway", "run"]
           env:
             - name: HERMES_DASHBOARD
@@ -144,12 +168,38 @@ spec:
               mountPath: /opt/data
             - name: tools
               mountPath: /opt/data/.local/bin
+            # Identity and house rules from git (SOUL.md beside this file).
+            - name: soul
+              mountPath: /opt/data/SOUL.md
+              subPath: SOUL.md
+              readOnly: true
+            - name: skills
+              mountPath: /opt/hermes-skills
+              readOnly: true
+            - name: wiki-key
+              mountPath: /var/run/secrets/wiki
+              readOnly: true
       volumes:
         - name: data
           persistentVolumeClaim:
             claimName: hermes-data
         - name: tools
           emptyDir: {}
+        - name: soul
+          configMap:
+            name: hermes-soul
+        - name: skills
+          configMap:
+            name: hermes-skills
+            items:
+              - key: homelab-wiki-SKILL.md
+                path: homelab-wiki/SKILL.md
+              - key: homelab-wiki-wiki.py
+                path: homelab-wiki/scripts/wiki.py
+        - name: wiki-key
+          secret:
+            secretName: hermes-wiki
+            optional: true
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/hermes/kustomization.yaml

@@ -3,5 +3,16 @@ kind: Kustomization
 resources:
   - rbac.yaml
   - api-key.yaml
+  - wiki-externalsecret.yaml
   - hermes.yaml
   - ingress.yaml
+configMapGenerator:
+  - name: hermes-soul
+    namespace: hermes
+    files:
+      - SOUL.md
+  - name: hermes-skills
+    namespace: hermes
+    files:
+      - homelab-wiki-SKILL.md=skills/homelab-wiki/SKILL.md
+      - homelab-wiki-wiki.py=skills/homelab-wiki/scripts/wiki.py

kubernetes/apps/hermes/skills/homelab-wiki/SKILL.md

@@ -0,0 +1,49 @@
+---
+name: homelab-wiki
+description: Ask and extend the homelab wiki, the owner's long-term knowledge base (services, hosts, network, past incidents, decisions, tool quirks). Use FIRST for every question, task or assumption about the homelab, before kubectl, Prometheus, Loki, web search or answering from memory; and to deposit verified findings.
+version: 1.0.0
+metadata:
+  hermes:
+    tags: [homelab, knowledge, wiki]
+    category: research
+---
+
+# Homelab wiki
+
+The wiki service (`http://wiki.ai.svc.cluster.local:8080`) owns the whole pipeline:
+raw sources → distilled notes. Talk to it only through
+`/opt/hermes-skills/homelab-wiki/scripts/wiki.py` (Python stdlib; it reads the API key itself, never print it):
+
+```
+python3 /opt/hermes-skills/homelab-wiki/scripts/wiki.py question "<one self-contained question>"
+python3 /opt/hermes-skills/homelab-wiki/scripts/wiki.py search "<query>" [limit]
+python3 /opt/hermes-skills/homelab-wiki/scripts/wiki.py note <slug>
+python3 /opt/hermes-skills/homelab-wiki/scripts/wiki.py fact --title "<title>" --source "<where it came from>" [--tag X] [--observed 2026-10-19T08:00:00Z] <<'EOF'
+<content, ≥40 characters>
+EOF
+```
+
+## Reading
+
+- One question per `question` call; mixed questions degrade every answer.
+- `covered` ≥ 0.7: trust the answer. 0.4–0.7: partial; use what the evidence
+  supports and say what is missing. Below 0.4 or `gap` set: the wiki does not
+  know. Say so; never fill the gap from training data.
+- Cite the `[[slug]]`s when relaying facts. Preserve disagreement the wiki
+  surfaces; don't silently pick a winner.
+- `search` + `note` when evidence is cut off or points at another note. Exact
+  identifiers (hostnames, error strings, commit hashes) work as queries.
+- Answers may come back in German; relay them in the user's language.
+
+## Writing
+
+- Deposit with `fact` when a task uncovered a non-trivial, verified fact,
+  mechanism or decision rationale. The service distills it into notes.
+- Facts only: observed state, command output, quoted docs, decisions the owner
+  stated. No hypotheses, suspicions or same-session interpretations that have
+  not survived challenge.
+- `--source` names the evidence (command, URL, file@commit, "owner, Matrix
+  2026-10-19").
+- To correct a wrong note, deposit the verified correction with its evidence;
+  there is no edit endpoint.
+- Never put secrets in a fact.

kubernetes/apps/hermes/skills/homelab-wiki/scripts/wiki.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Client for the homelab wiki service (in-cluster, API key auth).
+
+  wiki.py question "<question>"         answer with evidence and a coverage gate
+  wiki.py search "<query>" [limit]       ranked slugs + titles
+  wiki.py note <slug>                    full note
+  wiki.py fact --title T --source S [--tag X]... [--observed ISO] < content
+                                         deposit a raw source (the service distills it)
+"""
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+URL = os.environ.get("WIKI_URL", "http://wiki.ai.svc.cluster.local:8080")
+KEY_FILE = os.environ.get("WIKI_API_KEY_FILE", "/var/run/secrets/wiki/api-key")
+KEY = open(KEY_FILE).read().strip() if os.path.exists(KEY_FILE) else ""
+
+
+def call(method, path, body=None):
+    headers = {"Accept": "application/json"}
+    if KEY:
+        headers["X-Api-Key"] = KEY
+    data = None
+    if body is not None:
+        data = json.dumps(body).encode()
+        headers["Content-Type"] = "application/json"
+    req = urllib.request.Request(URL + path, data=data, headers=headers, method=method)
+    try:
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            return json.load(resp)
+    except urllib.error.HTTPError as e:
+        sys.exit(f"wiki: HTTP {e.code}\n{e.read().decode(errors='replace')}")
+
+
+def main():
+    p = argparse.ArgumentParser(prog="wiki.py")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    q = sub.add_parser("question")
+    q.add_argument("question")
+    s = sub.add_parser("search")
+    s.add_argument("query")
+    s.add_argument("limit", nargs="?", type=int, default=10)
+    n = sub.add_parser("note")
+    n.add_argument("slug")
+    f = sub.add_parser("fact")
+    f.add_argument("--title", required=True)
+    f.add_argument("--source", required=True)
+    f.add_argument("--tag", action="append", default=[])
+    f.add_argument("--observed")
+    a = p.parse_args()
+
+    if a.cmd == "question":
+        r = call("POST", "/api/v1/questions", {"question": a.question, "save": False})
+        out = {k: r.get(k) for k in ("answer", "gap", "covered")}
+        out["evidence"] = [{k: e.get(k) for k in ("slug", "heading", "text")} for e in r.get("evidence") or []]
+        print(json.dumps(out, indent=2, ensure_ascii=False))
+    elif a.cmd == "search":
+        qs = urllib.parse.urlencode({"q": a.query, "limit": a.limit})
+        for hit in call("GET", f"/api/v1/notes?{qs}"):
+            print(f"{str(hit['score'])[:5]}  {hit['slug']}  — {hit['title']}")
+    elif a.cmd == "note":
+        r = call("GET", "/api/v1/notes/" + urllib.parse.quote(a.slug, safe=""))
+        print(f"slug: {r['slug']}  updated: {r.get('updated')}")
+        print("sources:\n" + "\n".join("  - " + s["text"] for s in r.get("sources") or []))
+        print("related: " + ", ".join(r.get("related") or []))
+        print("backlinks: " + ", ".join(sorted({b["fromSlug"] for b in r.get("backlinks") or []})))
+        print()
+        print(r.get("body", ""))
+    elif a.cmd == "fact":
+        if not KEY:
+            sys.exit(f"wiki: fact needs the API key at {KEY_FILE}")
+        content = sys.stdin.read()
+        if len(content) < 40:
+            sys.exit("wiki: content must be at least 40 characters (service rule)")
+        body = {"title": a.title, "source": a.source, "kind": "fact", "content": content, "tags": a.tag}
+        if a.observed:
+            body["observed_at"] = a.observed
+        r = call("POST", "/api/v1/facts", body)
+        print(f"stored {r['path']} (commit {r['commit'][:8]}, by {r['capturedBy']})")
+
+
+if __name__ == "__main__":
+    main()

kubernetes/apps/hermes/wiki-externalsecret.yaml

@@ -0,0 +1,19 @@
+# Hermes's wiki API key (identity `hermes`); the wiki's copy is
+# apps/ai/wiki-api-key-hermes.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hermes-wiki
+  namespace: hermes
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hermes-wiki
+  data:
+    - secretKey: api-key
+      remoteRef:
+        key: apps/hermes/wiki
+        property: api_key

kubernetes/apps/kuma-sync/deployment.yaml

@@ -1,4 +1,3 @@
----
 # kuma-sync: KumaMonitor -> Uptime Kuma monitors on kuma.renner.uno (ci/images/kuma-sync).
 apiVersion: apps/v1
 kind: Deployment
@@ -31,7 +30,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: controller
-          image: gitea.renner.uno/gitea_admin/kuma-sync:2026-10-19-0545 # {"$imagepolicy": "flux-system:kuma-sync"}
+          image: gitea.renner.uno/gitea_admin/kuma-sync:2026-10-19-0615 # {"$imagepolicy": "flux-system:kuma-sync"}
           env:
             - name: KUMA_URL
               value: https://kuma.renner.uno

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - prometheusrule-netbird-availability.yaml
   - prometheusrule-netbird-ingress.yaml
   - prometheusrule-openbao.yaml
+  - prometheusrule-uptime-kuma.yaml
   # Prometheus's own retention. Until REN-107 `retention: 90d` and
   # `retentionSize: 30GB` were both set, the size cap bound first, and the
   # effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.

kubernetes/apps/monitoring-rules/probes-blackbox.yaml

@@ -44,6 +44,8 @@ spec:
         # traverses the WAN and detects a controller/DC outage (the
         # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200.
         - https://netbird.renner.uno
+        # Dead-man probe for the off-site Uptime Kuma.
+        - https://kuma.renner.uno
         # NetBird relay health endpoints on the netcup members (relay
         # binary serves :9000/health, verified on 0.73.2). Plain http,
         # ports outside the hairpin's 80/443 — real WAN path.

kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml

@@ -26,6 +26,7 @@ spec:
         #   proxy-1/2:9000/health         same file, relay redundancy
         #   ci.renner.uno                 WoodpeckerDown
         #   openbao .../sys/health        OpenBaoUnavailable
+        #   kuma.renner.uno               UptimeKumaUnreachable
         #
         # Warning, not critical, on purpose: this rule cannot know what a given
         # target means. A target whose loss deserves a critical page gets its
@@ -33,7 +34,7 @@ spec:
         # exclusion list from being arbitrary.
         # anchor: blackbox-probe-failing
         - alert: BlackboxProbeFailing
-          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
+          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://kuma\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
           for: 10m
           annotations:
             summary: "Blackbox probe of {{ $labels.instance }} has been failing for 10 minutes"

kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml

@@ -0,0 +1,35 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: uptime-kuma
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: uptime-kuma.rules
+      rules:
+        # Dead-man probe for the off-site Uptime Kuma.
+        - alert: UptimeKumaUnreachable
+          expr: probe_success{job="blackbox-http", instance="https://kuma.renner.uno"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
+            description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
+
+        # monitor_response_time is ms; -1 when a beat has no ping (DOWN, group).
+        # anchor: uptime-kuma-slow-response
+        - alert: UptimeKumaSlowResponse
+          expr: avg by (monitor_name) (avg_over_time((monitor_response_time > 0)[15m:1m])) > 1500
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Uptime Kuma monitor {{ $labels.monitor_name }} averaging {{ $value | humanize }} ms response time"
+            description: >-
+              The average response time of monitor {{ $labels.monitor_name }},
+              as measured from the off-site Uptime Kuma, has stayed above
+              1500 ms for 15 minutes.

kubernetes/apps/monitoring-rules/tests/README.md

@@ -32,6 +32,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
 | `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
 | `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
+| `uptime-kuma_test.yaml` | Hand-written suite for `prometheusrule-uptime-kuma.yaml` and the kuma exclusion in `prometheusrule-blackbox.yaml`. |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |
 
 ## Adding a suite

kubernetes/apps/monitoring-rules/tests/mutations.txt

@@ -172,3 +172,12 @@ disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusr
 # exactly-25% case in gitea_test.yaml.
 gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
 gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/
+
+# prometheusrule-uptime-kuma.yaml / blackbox kuma exclusion. Detected by the
+# cases in uptime-kuma_test.yaml.
+kuma-slow-no-positive-filter|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/(monitor_response_time > 0)/(monitor_response_time)/
+kuma-slow-threshold-2500|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 2500/
+kuma-slow-threshold-900|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 900/
+kuma-slow-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaSlowResponse/,/for:/s/for: 15m/for: 30m/
+kuma-down-for-20m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaUnreachable/,/for:/s/for: 10m/for: 20m/
+kuma-blackbox-exclusion-dropped|kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml|s/kuma\\\\\.renner/kumx\\\\.renner/

kubernetes/apps/monitoring-rules/tests/uptime-kuma_test.yaml

@@ -0,0 +1,110 @@
+# Suite for prometheusrule-uptime-kuma.yaml and the kuma exclusion in
+# prometheusrule-blackbox.yaml.
+rule_files:
+  - ../prometheusrule-uptime-kuma.yaml
+  - ../prometheusrule-blackbox.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # 2000 ms sustained: pending at 0m, fires 15m later, not before.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="slow-mon"}'
+        values: '2000+0x40'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+      - eval_time: 16m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              monitor_name: slow-mon
+            exp_annotations:
+              summary: "Uptime Kuma monitor slow-mon averaging 2k ms response time"
+              description: "The average response time of monitor slow-mon, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
+
+  # 1000 ms stays silent; a monitor reporting only -1 (DOWN, group) stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="fast-mon"}'
+        values: '1000+0x40'
+      - series: 'monitor_response_time{monitor_name="down-mon"}'
+        values: '-1+0x40'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+
+  # Alternating 1000 and -1: the -1 samples must not drag a slow average down
+  # nor count as slow; positives average 1000, silent.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="flap-fast"}'
+        values: '1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+
+  # Alternating 2000 and -1: the -1 samples are excluded, so the average of the
+  # real samples is 2000 and the alert fires. Unfiltered, the mean is ~500.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="flap-slow"}'
+        values: '2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              monitor_name: flap-slow
+            exp_annotations:
+              summary: "Uptime Kuma monitor flap-slow averaging 2k ms response time"
+              description: "The average response time of monitor flap-slow, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
+
+  # Dead-man probe: critical after 10m of failure; the catch-all warning must
+  # not also fire for kuma, but still fires for another failing instance.
+  - interval: 1m
+    input_series:
+      - series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
+        values: '0+0x40'
+      - series: 'probe_success{job="blackbox-http",instance="https://other.renner.uno"}'
+        values: '0+0x40'
+    alert_rule_test:
+      - eval_time: 9m
+        alertname: UptimeKumaUnreachable
+        exp_alerts: []
+      - eval_time: 11m
+        alertname: UptimeKumaUnreachable
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              job: blackbox-http
+              instance: https://kuma.renner.uno
+            exp_annotations:
+              summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
+              description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
+      - eval_time: 15m
+        alertname: BlackboxProbeFailing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              job: blackbox-http
+              instance: https://other.renner.uno
+            exp_annotations:
+              summary: "Blackbox probe of https://other.renner.uno has been failing for 10 minutes"
+              description: "The endpoint is not answering as expected. For the Authentik-gated hostnames this is a reachability check covering DNS, TLS, Traefik and the gate outpost — it says nothing about the app behind the gate, which the internal /-/healthy probes cover separately."
+
+  # Healthy probe stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
+        values: '1+0x40'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: UptimeKumaUnreachable
+        exp_alerts: []

kubernetes/apps/monitoring/helmrelease-prometheus.yaml

@@ -97,6 +97,10 @@ spec:
               resources:
                 requests:
                   storage: 50Gi
+        # Mounted at /etc/prometheus/secrets/<name>/; see
+        # ./uptime-kuma-metrics-externalsecret.yaml.
+        secrets:
+          - uptime-kuma-metrics
         # Additional scrape targets not covered by ServiceMonitors.
         additionalScrapeConfigs:
           # Gitea exposes /metrics anonymously on its main HTTP port. The
@@ -106,6 +110,16 @@ spec:
             static_configs:
               - targets: ["gitea.gitea.svc.cluster.local:3000"]
             metrics_path: /metrics
+          # Off-site Uptime Kuma. /metrics takes the API key as the basic-auth
+          # password with an empty username.
+          - job_name: uptime-kuma
+            scheme: https
+            metrics_path: /metrics
+            basic_auth:
+              username: ""
+              password_file: /etc/prometheus/secrets/uptime-kuma-metrics/api_key
+            static_configs:
+              - targets: ["kuma.renner.uno:443"]
 
     # Grafana
     grafana:

kubernetes/apps/monitoring/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
   - grafana-oauth-externalsecret.yaml
   - alertmanager-smtp-externalsecret.yaml
+  - uptime-kuma-metrics-externalsecret.yaml
   - helmrelease-prometheus.yaml
   - helmrelease-loki.yaml
   - helmrelease-alloy-syslog.yaml

kubernetes/apps/monitoring/uptime-kuma-metrics-externalsecret.yaml

@@ -0,0 +1,19 @@
+---
+# Uptime Kuma /metrics API key for Prometheus; OpenBao homelab/infrastructure/uptime-kuma-metrics.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: uptime-kuma-metrics
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: uptime-kuma-metrics
+  data:
+    - secretKey: api_key
+      remoteRef:
+        key: infrastructure/uptime-kuma-metrics
+        property: api_key

kubernetes/apps/wiki/externalsecrets.yaml

@@ -24,7 +24,8 @@ spec:
         key: apps/ai/wiki-index
         property: webhook_secret
 ---
-# One API key per machine identity: OpenBao apps/ai/wiki-api-keys, field = name.
+# One API key per machine identity: OpenBao apps/ai/wiki-api-keys, field = name;
+# hermes in apps/ai/wiki-api-key-hermes.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
@@ -42,6 +43,10 @@ spec:
       remoteRef:
         key: apps/ai/wiki-api-keys
         property: workstation
+    - secretKey: Auth__ApiKeys__hermes
+      remoteRef:
+        key: apps/ai/wiki-api-key-hermes
+        property: api_key
 ---
 # TypeSafe System One key for the judge adapter (Wiki:Judges:Models:typesafe).
 apiVersion: external-secrets.io/v1

kubernetes/infrastructure/image-automation/hermes.yaml

@@ -8,6 +8,9 @@ spec:
   interval: 1h
 ---
 # v-prefix tags (v0.22.2). The filter skips rc.*, -desktop and per-arch tags.
+# The repo also carries the retired date-versioned releases (v2026.4.3 ..
+# v2026.9.24, last one 2026-09-24); as semver they sort above 0.x, so the
+# range caps below 1.0.0 or the policy picks the old v2026.9.24.
 apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImagePolicy
 metadata:
@@ -21,4 +24,4 @@ spec:
     extract: '$ver'
   policy:
     semver:
-      range: ">=0.22.2"
+      range: ">=0.22.2 <1.0.0"

kubernetes/infrastructure/network-policies/ai-ns.yaml

@@ -141,8 +141,8 @@ spec:
               protocol: TCP
 ---
 # wiki: Traefik (the machine door wiki-index.renner.uno, BasicAuth with
-# X-Authentik-Username stripped) and Gitea (push webhook) may reach it; the
-# gate outpost (humans, after Authentik) is admitted
+# X-Authentik-Username stripped), Gitea (push webhook) and Hermes Agent (API
+# key) may reach it; the gate outpost (humans, after Authentik) is admitted
 # by the CNP wiki-gate that authentik-gate-sync generates. Gitea and wiki-embed
 # are reached through the namespace egress above; no OpenBao access.
 apiVersion: cilium.io/v2
@@ -162,6 +162,9 @@ spec:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": gitea
             app.kubernetes.io/name: gitea
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": hermes
+            app.kubernetes.io/name: hermes
       toPorts:
         - ports:
             - port: "8080"

kubernetes/infrastructure/network-policies/hermes-ns.yaml

@@ -113,6 +113,15 @@ spec:
         - ports:
             - port: "3100"
               protocol: TCP
+    # The wiki service API; ingress side is the `wiki` CNP in ai-ns.yaml.
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ai
+            app.kubernetes.io/name: wiki
+      toPorts:
+        - ports:
+            - port: "8080"
+              protocol: TCP
 ---
 # Traefik serves the dashboard (9119) and the OpenAI-compatible API (8642).
 apiVersion: cilium.io/v2

kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml

@@ -23,6 +23,7 @@ spec:
         - evcc
         - flux-system # flux-system-git-auth: syncs the rotated flux gitea PAT into Flux's git secret
         - gitea # gitea-package-retention token (apps/gitea/package-retention.yaml, REN-1203)
+        - hermes # hermes-wiki (apps/hermes/wiki-externalsecret.yaml)
         - home-assistant
         - immich
         - influxdb