Merge pull request 'feat(ansible): netbird sshd listens on the NetBird interface only' (#1492 ) from ssh-mesh-only-netbird into main

Reviewed-on: #1492
feat(ansible): netbird sshd listens on the NetBird interface only
2026-10-27 18:29:26 +00:00 · 2026-10-27 19:26:30 +01:00 · 2026-10-27 18:25:31 +00:00 · 2026-10-27 19:24:07 +01:00 · 2026-10-27 18:17:57 +00:00 · 2026-10-27 19:14:10 +01:00
64 changed files with 835 additions and 491 deletions
@@ -35,25 +35,25 @@ when:
 steps:
  - name: model-selection-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    commands:
      - sh scripts/ci/test-model-selection.sh

  - name: pr-review-mistral-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    commands:
      - sh scripts/ci/test-pr-review-mistral.sh

  - name: promtool-rules-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    commands:
      - sh scripts/ci/test-promtool-rules.sh

  - name: yaml-citations-guard-tests
    depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    commands:
      - sh scripts/ci/test-yaml-citations.sh

@@ -1,33 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/claude-workstation/**"
-      - "ansible/plays/install-claude-workstation.yml"
-      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
-
-timeout: 30
-
-steps:
-  - name: deploy-claude-workstation
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-claude-workstation.yml -v
@@ -1,36 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/mqtt-bridge/**"
-      - "ansible/plays/edge-mqtt-bridge.yml"
-      - "ansible/inventory/group_vars/edge_host/**"
-      - ".woodpecker/deploy-edge-mqtt-bridge.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-edge-mqtt-bridge"'
-
-timeout: 30
-
-steps:
-  - name: deploy-edge-mqtt-bridge
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      # Local MQTT bridge on the edge hosts only (REN-1307); restarts only the
-      # local mosquitto on edge-1, never the cluster broker.
-      - ansible-playbook -i inventory/ plays/edge-mqtt-bridge.yml -v
@@ -109,8 +109,6 @@ steps:
        from_secret: hetzner_s3_access_key
      TF_VAR_hetzner_s3_secret_key:
        from_secret: hetzner_s3_secret_key
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
      PROXMOX_TOKEN_ID:
        from_secret: proxmox_token_id
      PROXMOX_TOKEN_SECRET:
@@ -121,11 +119,6 @@ steps:
      ANSIBLE_ROLES_PATH: "../roles:./roles"
      ANSIBLE_FORCE_COLOR: "true"
    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - export TF_VAR_ssh_private_key="$(cat ~/.ssh/id_ed25519)"
-      - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
      - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
      - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
      - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"
@@ -1,35 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-client/**"
-      - "ansible/plays/install-netbird-client.yml"
-      - "ansible/inventory/group_vars/claude_workstation/**"
-      - ".woodpecker/deploy-netbird-client.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-client
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      # Enrol only the off-site Claude workstation into the Netbird mesh.
-      - ansible-playbook -i inventory/ plays/install-netbird-client.yml --limit claude_workstation -v
@@ -1,54 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-proxy/**"
-      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
-      # so changes to it must redeploy this host too.
-      - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
-      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
-      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-proxy
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-netbird-proxy.yml -v
@@ -1,37 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-server/**"
-      - "ansible/plays/install-netbird-server.yml"
-      - ".woodpecker/deploy-netbird-server.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-server
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-netbird-server.yml -v
@@ -1,13 +1,4 @@
 when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/uptime-kuma/**"
-      - "ansible/plays/install-uptime-kuma.yml"
-      - ".woodpecker/deploy-uptime-kuma.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-uptime-kuma"'
  - event: cron
    cron: uptime-kuma-token-renew

@@ -21,27 +12,3 @@ steps:
        from_secret: openbao_uptime_kuma_token
    commands:
      - 'curl -sSf -X POST -H "X-Vault-Token: $VAULT_TOKEN" https://openbao.renner.uno/v1/auth/token/renew-self -o /dev/null'
-
-  - name: deploy-uptime-kuma
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-10-18
-    when:
-      - event: [push, deployment]
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      VAULT_TOKEN:
-        from_secret: openbao_uptime_kuma_token
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-uptime-kuma.yml -v
@@ -26,7 +26,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -70,7 +70,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: main
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -30,7 +30,7 @@ when:

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,28 +0,0 @@
-when:
-  # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the
-  # NetBird control plane. Run inside the 02:00-05:00 Europe/Berlin window.
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"'
-
-timeout: 120
-
-steps:
-  - name: update-os
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/update-os.yml -v
@@ -206,7 +206,7 @@ inventory=~/source/homelab/ansible/inventory
 ;poll_interval=15

 # (path) Option for connections using a certificate or key file to authenticate, rather than an agent or passwords, you can set the default value here to avoid re-specifying --private-key with every invocation.
-private_key_file=~/.ssh/id_ed25519
+;private_key_file=

 # (boolean) By default, imported roles publish their variables to the play and other roles, this setting can avoid that.
 # This was introduced as a way to reset role variables to default values if a role is used more than once in a playbook.
@@ -245,8 +245,8 @@ its own server key, nothing else. On a push it writes
 `/etc/mosquitto/bridge.d/edge-1.conf`, deletes `bridge.d/legacy.conf` and
 restarts the local mosquitto. The controller sets a new password whenever the
 bridge reports "not connected" (also after a reinstall), so no play run is
-needed for a rotation. `.woodpecker/deploy-edge-mqtt-bridge.yml` runs
-`plays/edge-mqtt-bridge.yml` on merge. Diagnosis: `journalctl -u
+needed for a rotation. `plays/edge-mqtt-bridge.yml` is run by hand from a
+workstation. Diagnosis: `journalctl -u
 edge-mqtt-cred`, and `mosquitto_sub -t '$SYS/broker/connection/edge-1/state'`
 (retained `1` = connected).

@@ -5,27 +5,10 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3

-# SSH Public Keys for Cloud-Init
-# ansible-runner@ci-new removed (issue #184): dead key, no such host exists.
-common_ssh_public_keys: |
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-
-# Network-of-origin restriction for the woodpecker-ci@homelab CI key
-# (issue #184). Consumed by plays/configure-ssh-keys.yml as the authorized_key
-# `from=` option so the CI key only authenticates from within the cluster's
-# pod network — NOT from any host-pinned IP.
-#
-# The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
-# source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +
-# `ipv6-native-routing-cidr=fd00::/8` with masquerade enabled, so pod egress to
-# any 10.x / fd.. LAN destination is NOT SNAT'd, and each node's PodCIDR is
-# advertised to the UDM over BGP (kubernetes/infrastructure/post-crds/
-# cilium-bgp.yaml) so return traffic L3-routes back to the pod. Hosts therefore
-# see the real pod source IP.
-#
-# The cluster pod subnets (kubernetes/talos/patches/cluster.yaml podSubnets)
-# are the stable boundary — they survive node DHCP-lease churn and pod
-# rescheduling across k8s-1/2/3, unlike pinning any node's current IP.
-# v6 is included because some SSH targets (edge IoT hosts) are IPv6-only.
-ci_ssh_from_cidrs: '10.244.0.0/16,fd10:244::/48'
+# The only plain key the admin logins accept on [ssh_keys_managed] hosts
+# (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
+# certificates; this one is for when that path is unavailable.
+#   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
+#     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
+ssh_admin_authorized_keys:
+  - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
@@ -0,0 +1,4 @@
+---
+# The host NetBird client (wt0) must not take over DNS: resolv.conf is pinned to
+# the systemd-resolved stub by the netbird-proxy role.
+netbird_disable_dns: true
@@ -13,11 +13,11 @@
 # Their hosts no longer resolve, except the three that resolve to Talos nodes.

 [netbird_server]
-netbird.renner.uno ansible_user=root
+netbird.renner.uno ansible_host=netbird.nb.renner.uno ansible_user=root

 [netbird_proxy]
-proxy-1.renner.uno ansible_user=root
-proxy-2.renner.uno ansible_user=root
+proxy-1.renner.uno ansible_host=proxy-1.nb.renner.uno ansible_user=root
+proxy-2.renner.uno ansible_host=proxy-2.nb.renner.uno ansible_user=root

 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
@@ -69,3 +69,29 @@ edge_host
 # home, so it must NOT be used as ansible_host.
 [claude_workstation]
 claude-ws ansible_host=10.3.40.78 ansible_user=root
+
+# Hosts that trust the OpenBao SSH user CA (plays/configure-ssh-ca.yml).
+# Rolled out one host at a time: add the next host after the previous one
+# was verified.
+[ssh_ca]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno
+edge-1.iot.renner.uno
+
+# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass);
+# everything else needs a CA certificate. Members must be in [ssh_ca].
+# Added one host at a time after the [ssh_ca] report was reviewed.
+[ssh_keys_managed]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno
+edge-1.iot.renner.uno
+
+# Internet hosts whose sshd listens on the NetBird interface only
+# (plays/configure-ssh-mesh-only.yml). Members need the host NetBird client.
+# Added one host at a time after the previous one was verified.
+[ssh_mesh_only]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno
@@ -0,0 +1,23 @@
+---
+# Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
+# owner certificates (principal k.renner) for root/ladmin, Claude's 1h
+# certificates (principal claude-bot) for its own claude-bot account with
+# limited sudo. The role reports the admin logins' authorized_keys; on
+# [ssh_keys_managed] they are then reduced to ssh_admin_authorized_keys.
+#
+# serial: 1 so a host whose sshd rejects the change stops the run before the
+# next host is touched. One play, so each host needs one login (one YubiKey
+# PIN from a workstation).
+- name: Configure the SSH user CA
+  hosts: ssh_ca
+  become: true
+  gather_facts: true
+  serial: 1
+  roles:
+    - ssh-ca
+  tasks:
+    - name: Reduce authorized_keys to the break-glass key
+      ansible.builtin.include_role:
+        name: ssh-ca
+        tasks_from: authorized_keys
+      when: inventory_hostname in groups['ssh_keys_managed']
@@ -1,48 +0,0 @@
---
- name: Deploy authorized SSH keys to all hosts
-  hosts: all:!localhost
-  become: true
-  gather_facts: false
-  # Authorized keys deployed to root and ladmin on every managed host.
-  #
-  # - krenner@fortress (human): unrestricted.
-  # - woodpecker-ci@homelab (CI): restricted with from= to the cluster's pod
-  #   network (ci_ssh_from_cidrs in group_vars/all/vars.yml). The in-cluster
-  #   Woodpecker step pods SSH to fleet hosts presenting their POD IP as source
-  #   (Cilium native-routing-cidr covers the LAN, so no SNAT; PodCIDRs are
-  #   BGP-advertised). This is a network-of-origin restriction, NOT host-pinned,
-  #   so it survives node DHCP churn and pod rescheduling. See vars.yml for the
-  #   full derivation.
-  # - ansible-runner@ci-new: removed — dead key, no such host exists.
-  vars:
-    authorized_keys:
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
-        options: 'from="{{ ci_ssh_from_cidrs }}"'
-  tasks:
-    - name: Check SSH connectivity
-      ansible.builtin.wait_for_connection:
-        timeout: 5
-      ignore_errors: true
-      register: ssh_check
-
-    - name: Skip unreachable hosts
-      ansible.builtin.meta: end_host
-      when: ssh_check is failed
-
-    - name: Deploy authorized keys to root
-      ansible.builtin.authorized_key:
-        user: root
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-
-    - name: Deploy authorized keys to ladmin
-      ansible.builtin.authorized_key:
-        user: ladmin
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-      ignore_errors: true
@@ -0,0 +1,15 @@
+---
+# sshd on the hosts in [ssh_mesh_only] accepts connections only on the NetBird
+# interface; nothing listens on their public addresses. Run from a workstation
+# in the mesh. A switchover reverts by itself unless the play logs in again
+# through the new listener.
+#
+# serial: 1 so a host that does not come back stops the run before the next
+# host is touched.
+- name: Restrict sshd to the mesh interface
+  hosts: ssh_mesh_only
+  become: true
+  gather_facts: true
+  serial: 1
+  roles:
+    - ssh-mesh-only
@@ -1,8 +1,6 @@
 ---
-# Local MQTT bridge and its credential receiver only (REN-1307). Run by
-# .woodpecker/deploy-edge-mqtt-bridge.yml on merge, so a change to this role
-# reaches edge-1 without a play run by hand. The full base play
-# (install-edge-host.yml) carries the same role.
+# Local MQTT bridge and its credential receiver only (REN-1307). Run from a
+# workstation. The full base play (install-edge-host.yml) carries the same role.
 - name: Converge the edge MQTT bridge
  hosts: edge_host
  become: true
@@ -15,17 +15,20 @@
 - name: Install NetBird client on hosts with setup key defined
  hosts: all
  become: true
-  gather_facts: true
+  # One host at a time: from a workstation every new SSH session asks for the
+  # YubiKey PIN and a touch, and parallel hosts would prompt at once.
+  serial: 1
+  # Facts only after the skip: gathering them first opens an SSH session (a
+  # YubiKey touch from a workstation) to every host, including skipped ones.
+  gather_facts: false

  pre_tasks:
-    - name: Check if netbird_setup_key is defined
-      ansible.builtin.debug:
-        msg: "NetBird setup key is defined for {{ inventory_hostname }}"
-      when: netbird_setup_key is defined
-
    - name: Skip hosts without netbird_setup_key
      ansible.builtin.meta: end_host
      when: netbird_setup_key is not defined

+    - name: Gather facts
+      ansible.builtin.setup:
+
  roles:
    - netbird-client
@@ -51,10 +51,8 @@
 # -------------------------------------------------------
 # 4. SSH Configuration
 # -------------------------------------------------------
-# Authorized keys (human + woodpecker-ci, the latter from=-restricted) are
-# managed by the dedicated ansible/plays/configure-ssh-keys.yml play. The old
-# "Add Ansible Runner SSH Key" task deployed ansible-runner@ci-new, a dead key
-# for a host that no longer exists — removed (issue #184).
+# SSH access (CA trust, admin authorized_keys) is managed by the ssh-ca role
+# via plays/configure-ssh-ca.yml.

 # -------------------------------------------------------
 # 5. Disable NTP on VMs/LXC (use host time)
@@ -6,3 +6,13 @@ netbird_management_url: "https://netbird.renner.uno"

 # Setup key - should be defined in group_vars or host_vars
 # netbird_setup_key: ""
+
+# Release installed on hosts without a NetBird package repository (Arch).
+# Debian/Ubuntu follow the apt repository instead.
+netbird_client_version: "0.75.0"
+netbird_release_url: "https://github.com/netbirdio/netbird/releases/download/v{{ netbird_client_version }}"
+# On the controller; the archive is copied from there to the hosts.
+netbird_download_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}/.cache/ansible/netbird"
+
+# Leave the host's DNS configuration alone (hosts with a pinned resolv.conf).
+netbird_disable_dns: false
@@ -2,32 +2,82 @@
 # NetBird Client Installation Role
 # Installs and configures NetBird client on hosts with netbird_setup_key defined

- name: Install required packages
-  ansible.builtin.apt:
-    name:
-      - curl
-      - ca-certificates
-      - gnupg
-    state: present
-    update_cache: true
+- name: Install NetBird from the NetBird apt repository
+  when: ansible_facts['os_family'] == 'Debian'
+  block:
+    - name: Install required packages
+      ansible.builtin.apt:
+        name:
+          - curl
+          - ca-certificates
+          - gnupg
+        state: present
+        update_cache: true

- name: Add NetBird repository key
-  ansible.builtin.shell:
-    cmd: |
-      curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
-    creates: /usr/share/keyrings/netbird-archive-keyring.gpg
+    - name: Add NetBird repository key
+      ansible.builtin.shell:
+        cmd: |
+          curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
+        creates: /usr/share/keyrings/netbird-archive-keyring.gpg

- name: Add NetBird repository
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
-    state: present
-    filename: netbird
+    - name: Add NetBird repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
+        state: present
+        filename: netbird

- name: Install NetBird client
-  ansible.builtin.apt:
+    - name: Install NetBird client
+      ansible.builtin.apt:
+        name: netbird
+        state: present
+        update_cache: true
+
+# Arch has no netbird package in its repositories; the release binary is
+# verified against the release's checksums file. It is downloaded on the
+# controller and copied over: github.com has no IPv6 address, and proxy-1 is
+# IPv6-only.
+- name: Install NetBird from the release tarball
+  when: ansible_facts['os_family'] == 'Archlinux'
+  block:
+    - name: Create the controller download directory
+      ansible.builtin.file:
+        path: "{{ netbird_download_dir }}"
+        state: directory
+        mode: "0755"
+      delegate_to: localhost
+      become: false
+      run_once: true
+
+    - name: Download the NetBird release on the controller
+      ansible.builtin.get_url:
+        url: "{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        checksum: "sha256:{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_checksums.txt"
+        mode: "0644"
+      delegate_to: localhost
+      become: false
+      run_once: true
+
+    - name: Install the NetBird binary
+      ansible.builtin.unarchive:
+        src: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: /usr/local/bin
+        include:
+          - netbird
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the NetBird service
+      ansible.builtin.command: netbird service install
+      args:
+        creates: /etc/systemd/system/netbird.service
+
+- name: Enable NetBird service
+  ansible.builtin.service:
    name: netbird
-    state: present
-    update_cache: true
+    state: started
+    enabled: true

 - name: Check if NetBird is already connected
  ansible.builtin.command: netbird status
@@ -35,20 +85,17 @@
  changed_when: false
  failed_when: false

+# "Disconnected" contains "Connected", so match the management line.
 - name: Connect NetBird to management server
  ansible.builtin.command: >
    netbird up
    --management-url {{ netbird_management_url }}
    --setup-key {{ netbird_setup_key }}
-  when: "'Connected' not in netbird_status.stdout"
+    {{ '--disable-dns' if netbird_disable_dns else '' }}
+  when: "'Management: Connected' not in netbird_status.stdout"
  register: netbird_connect
  changed_when: netbird_connect.rc == 0
-
- name: Enable NetBird service
-  ansible.builtin.service:
-    name: netbird
-    state: started
-    enabled: true
+  no_log: true

 - name: Get NetBird status
  ansible.builtin.command: netbird status
@@ -136,23 +136,13 @@
    - /opt/netbird/proxy
    - /opt/netbird/proxy-state

-# The management store has no cluster-side backup. Pulling one put a root
-# key on this host that the cluster held, and this server's own state is
-# its own business to protect rather than the cluster's. Dropping the two
-# tasks that placed them would only stop ansible managing the key and the
-# script — the pair below revokes them instead, and can go once the role
-# has run against every host that ever carried them.
+# The management store has no cluster-side backup; the cluster's backup
+# key is gone with the managed authorized_keys (ssh-ca role).
 - name: Remove the backup dump script
  ansible.builtin.file:
    path: /usr/local/bin/netbird-backup-dump
    state: absent

- name: Revoke the cluster backup key
-  ansible.posix.authorized_key:
-    user: root
-    key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster"
-    state: absent
-
 - name: Wait for management container to be ready
  ansible.builtin.pause:
    seconds: 10
@@ -0,0 +1,18 @@
+---
+# SSH user CA (OpenBao ssh-client-signer). Hosts trust certificates signed by
+# it; the public key is fetched from OpenBao on the controller at run time.
+ssh_ca_public_key_url: "https://openbao.renner.uno/v1/ssh-client-signer/public_key"
+
+# Owner certificates (role owner) carry this principal. It is accepted for
+# whichever admin login the distro has; users that do not exist are skipped.
+ssh_ca_owner_principal: k.renner
+ssh_ca_owner_users:
+  - root
+  - ladmin
+
+# Claude's own account. Certificates (role claude, 1h) carry the principal.
+ssh_ca_claude_user: claude-bot
+ssh_ca_claude_principal: claude-bot
+
+# Units claude-bot may restart/reload via sudo on this host, e.g. [netbird].
+ssh_ca_claude_sudo_units: []
@@ -0,0 +1,2 @@
+# Public keys whose certificates sshd must reject (one per line, OpenSSH format).
+# Revoking a key also revokes every certificate issued for it.
@@ -0,0 +1,23 @@
+---
+# Replaces the admin logins' authorized_keys with ssh_admin_authorized_keys
+# (break-glass + CI). Everything else logs in with a CA certificate.
+- name: Look up the admin logins
+  ansible.builtin.getent:
+    database: passwd
+    key: "{{ item }}"
+    fail_key: false
+  loop: "{{ ssh_ca_owner_users }}"
+  register: ssh_ca_admin_lookup
+
+- name: Set the admin logins' authorized_keys to the managed list
+  ansible.posix.authorized_key:
+    user: "{{ item.item }}"
+    key: |
+      {% for k in ssh_admin_authorized_keys %}
+      {{ (k.options ~ ' ') if (k.options | default('')) else '' }}{{ k.key }}
+      {% endfor %}
+    exclusive: true
+  loop: "{{ ssh_ca_admin_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
@@ -0,0 +1,156 @@
+---
+- name: Fetch the SSH CA public key from OpenBao
+  ansible.builtin.uri:
+    url: "{{ ssh_ca_public_key_url }}"
+    return_content: true
+  delegate_to: localhost
+  become: false
+  run_once: true
+  register: ssh_ca_key
+
+- name: Check the fetched CA key is an SSH public key
+  ansible.builtin.assert:
+    that:
+      - ssh_ca_key.content is match('^ssh-ed25519 AAAA')
+    fail_msg: "Unexpected CA key from {{ ssh_ca_public_key_url }}"
+  run_once: true
+
+- name: Install the trusted user CA key
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_key.content | trim }}\n"
+    dest: /etc/ssh/trusted_user_ca_keys
+    owner: root
+    group: root
+    mode: "0644"
+
+# sshd refuses all public key logins when RevokedKeys is set but unreadable,
+# so the file is always deployed, even when empty.
+- name: Install the revoked keys list
+  ansible.builtin.copy:
+    src: revoked_keys
+    dest: /etc/ssh/revoked_keys
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Create the Claude account
+  ansible.builtin.user:
+    name: "{{ ssh_ca_claude_user }}"
+    shell: /bin/bash
+    create_home: true
+    password: "!"
+
+- name: Look up the owner admin logins
+  ansible.builtin.getent:
+    database: passwd
+    key: "{{ item }}"
+    fail_key: false
+  loop: "{{ ssh_ca_owner_users }}"
+  register: ssh_ca_owner_lookup
+
+- name: Create the principals directory
+  ansible.builtin.file:
+    path: /etc/ssh/auth_principals
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Accept the owner principal for the existing admin logins
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_owner_principal }}\n"
+    dest: "/etc/ssh/auth_principals/{{ item.item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ ssh_ca_owner_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
+
+- name: Accept the Claude principal for the Claude account
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_claude_principal }}\n"
+    dest: "/etc/ssh/auth_principals/{{ ssh_ca_claude_user }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Install the Claude sudo rules
+  ansible.builtin.template:
+    src: sudoers-claude.j2
+    dest: "/etc/sudoers.d/{{ ssh_ca_claude_user }}"
+    owner: root
+    group: root
+    mode: "0440"
+    validate: visudo -cf %s
+
+# A broken drop-in only bites on the next sshd restart, so it is tested here
+# and removed again if the full config does not parse.
+- name: Configure sshd for the user CA
+  block:
+    - name: Install the sshd drop-in
+      ansible.builtin.template:
+        src: ssh-ca.conf.j2
+        dest: /etc/ssh/sshd_config.d/50-ssh-ca.conf
+        owner: root
+        group: root
+        mode: "0644"
+
+    - name: Test the full sshd config
+      ansible.builtin.command: sshd -t
+      changed_when: false
+  rescue:
+    - name: Remove the drop-in that broke sshd -t
+      ansible.builtin.file:
+        path: /etc/ssh/sshd_config.d/50-ssh-ca.conf
+        state: absent
+
+    - name: Fail the host
+      ansible.builtin.fail:
+        msg: "sshd -t failed with the SSH CA drop-in; it was removed again"
+
+- name: Read the effective sshd settings
+  ansible.builtin.command: sshd -T
+  changed_when: false
+  register: ssh_ca_effective
+
+# Keyword case in sshd -T output differs between OpenSSH versions (mixed case
+# on 10.6), so the lines are compared lowercased.
+- name: Check sshd uses the CA, principals and revocation list
+  vars:
+    ssh_ca_effective_lines: "{{ ssh_ca_effective.stdout_lines | map('lower') | list }}"
+  ansible.builtin.assert:
+    that:
+      - "'trustedusercakeys /etc/ssh/trusted_user_ca_keys' in ssh_ca_effective_lines"
+      - "'authorizedprincipalsfile /etc/ssh/auth_principals/%u' in ssh_ca_effective_lines"
+      - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective_lines"
+    fail_msg: "sshd -T does not show the SSH CA settings; another config line takes precedence"
+
+# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu.
+# Ubuntu also lists sshd.service as a not-found alias, so the running one is
+# picked. sshd -T only parses the files, so the running daemon is reloaded on
+# every run; a reload keeps open sessions. None runs when sshd is socket
+# activated and stopped, or per connection (ssh-mesh-only role); the next
+# start reads the config.
+- name: Read the service list
+  ansible.builtin.service_facts:
+
+- name: Reload the running sshd unit
+  vars:
+    ssh_ca_sshd_units: "{{ ansible_facts.services | dict2items | selectattr('key', 'in', ['sshd.service', 'ssh.service']) | selectattr('value.state', 'equalto', 'running') | map(attribute='key') | list }}"
+  block:
+    - name: Check at most one sshd unit is running
+      ansible.builtin.assert:
+        that:
+          - ssh_ca_sshd_units | length <= 1
+        fail_msg: "Expected at most one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
+
+    - name: Reload sshd so the running daemon uses the CA config
+      ansible.builtin.systemd:
+        name: "{{ ssh_ca_sshd_units | first }}"
+        state: reloaded
+      when: ssh_ca_sshd_units | length == 1
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.include_tasks: report.yml
@@ -0,0 +1,26 @@
+---
+# Read-only: what the admin logins accept besides certificates. Hosts in
+# [ssh_keys_managed] have this reduced to ssh_admin_authorized_keys.
+- name: Find the admin logins' authorized_keys
+  ansible.builtin.stat:
+    path: "{{ item.ansible_facts.getent_passwd[item.item][4] }}/.ssh/authorized_keys"
+  loop: "{{ ssh_ca_owner_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
+  register: ssh_ca_keyfiles
+
+- name: Fingerprint the admin logins' authorized keys
+  ansible.builtin.command: ssh-keygen -lf {{ item.stat.path }}
+  loop: "{{ ssh_ca_keyfiles.results | selectattr('stat', 'defined') | selectattr('stat.exists') | list }}"
+  loop_control:
+    label: "{{ item.stat.path }}"
+  changed_when: false
+  register: ssh_ca_fingerprints
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.debug:
+    msg: "{{ item.item.stat.path }}: {{ item.stdout_lines }}"
+  loop: "{{ ssh_ca_fingerprints.results }}"
+  loop_control:
+    label: "{{ item.item.stat.path }}"
@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+TrustedUserCAKeys /etc/ssh/trusted_user_ca_keys
+AuthorizedPrincipalsFile /etc/ssh/auth_principals/%u
+RevokedKeys /etc/ssh/revoked_keys
@@ -0,0 +1,6 @@
+# {{ ansible_managed }}
+# NOEXEC: journalctl/systemctl open a pager, which must not spawn a shell.
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/journalctl, /usr/bin/journalctl *, /usr/bin/systemctl status, /usr/bin/systemctl status *, /usr/bin/systemctl list-units, /usr/bin/systemctl list-units *, /usr/bin/ss -tulpn, /usr/sbin/sshd -T
+{% for unit in ssh_ca_claude_sudo_units %}
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/systemctl restart {{ unit }}, /usr/bin/systemctl reload {{ unit }}
+{% endfor %}
@@ -0,0 +1,9 @@
+---
+# The only interface sshd accepts connections on.
+ssh_mesh_interface: wt0
+
+# Minutes until a switchover reverts to the distro listener, unless a new
+# login through the mesh-only listener confirms it first.
+ssh_mesh_revert_minutes: 5
+
+ssh_mesh_udev_rule: /etc/udev/rules.d/90-ssh-mesh-only.rules
@@ -0,0 +1,113 @@
+---
+- name: Load the distro's sshd unit names
+  ansible.builtin.include_vars: "{{ ansible_facts['os_family'] }}.yml"
+
+- name: Look for the mesh interface
+  ansible.builtin.stat:
+    path: "/sys/class/net/{{ ssh_mesh_interface }}"
+  register: ssh_mesh_iface
+
+- name: Check the mesh interface exists
+  ansible.builtin.assert:
+    that:
+      - ssh_mesh_iface.stat.exists
+    fail_msg: "No {{ ssh_mesh_interface }}: sshd would be unreachable. Install the NetBird client first."
+
+- name: Create the socket unit directory
+  ansible.builtin.file:
+    path: "{{ ssh_mesh_socket_file | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the mesh-only socket
+  ansible.builtin.template:
+    src: "{{ ssh_mesh_socket_template }}"
+    dest: "{{ ssh_mesh_socket_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Bind the sshd service to the socket
+  when: ssh_mesh_service_dropin | length > 0
+  block:
+    - name: Create the service drop-in directory
+      ansible.builtin.file:
+        path: "{{ ssh_mesh_service_dropin | dirname }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the service drop-in
+      ansible.builtin.template:
+        src: ssh.service-dropin.conf.j2
+        dest: "{{ ssh_mesh_service_dropin }}"
+        owner: root
+        group: root
+        mode: "0644"
+
+- name: Install the udev rule that starts the socket with the interface
+  ansible.builtin.template:
+    src: udev.rules.j2
+    dest: "{{ ssh_mesh_udev_rule }}"
+    owner: root
+    group: root
+    mode: "0644"
+  register: ssh_mesh_udev
+
+- name: Reload the udev rules
+  ansible.builtin.command: udevadm control --reload
+  when: ssh_mesh_udev is changed
+  changed_when: true
+
+- name: Install the switch and revert scripts
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/usr/local/sbin/{{ item }}"
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - ssh-mesh-switch
+    - ssh-mesh-revert
+
+- name: Reload systemd units
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Read the sshd listeners
+  ansible.builtin.command: ss -Htln 'sport = :22'
+  changed_when: false
+  register: ssh_mesh_listeners
+
+# Each run that finds a listener outside the interface switches over. An
+# include, because meta reset_connection ignores when: and a reconnect costs a
+# YubiKey PIN from a workstation.
+- name: Switch sshd to the mesh-only socket
+  ansible.builtin.include_tasks: switch.yml
+  when: >-
+    ssh_mesh_listeners.stdout_lines | length == 0 or
+    ssh_mesh_listeners.stdout_lines | reject('search', '%' ~ ssh_mesh_interface ~ ':22\s') | list | length > 0
+
+- name: Check sshd listens on the mesh interface only
+  ansible.builtin.assert:
+    that:
+      - ssh_mesh_listeners.stdout_lines | length > 0
+      - ssh_mesh_listeners.stdout_lines | reject('search', '%' ~ ssh_mesh_interface ~ ':22\s') | list | length == 0
+    fail_msg: "Listeners outside {{ ssh_mesh_interface }}: {{ ssh_mesh_listeners.stdout_lines }}"
+
+- name: Cancel the revert timer
+  ansible.builtin.command: systemctl stop ssh-mesh-revert.timer
+  changed_when: false
+  failed_when: false
+
+- name: Check the public name refuses SSH
+  ansible.builtin.wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 22
+    state: stopped
+    timeout: 15
+  delegate_to: localhost
+  become: false
@@ -0,0 +1,27 @@
+---
+# The revert timer runs in its own unit, so a switch that locks the play out is
+# undone after ssh_mesh_revert_minutes. main.yml cancels it after checking the
+# listeners read through the new one.
+- name: Clear a revert timer left by an earlier run
+  ansible.builtin.command: systemctl stop ssh-mesh-revert.timer ssh-mesh-revert.service
+  changed_when: false
+  failed_when: false
+
+- name: Arm the revert timer
+  ansible.builtin.command: >-
+    systemd-run --unit=ssh-mesh-revert --collect
+    --on-active={{ ssh_mesh_revert_minutes }}min
+    /usr/local/sbin/ssh-mesh-revert
+  changed_when: true
+
+- name: Switch the listener
+  ansible.builtin.command: systemd-run --unit=ssh-mesh-switch --collect --wait /usr/local/sbin/ssh-mesh-switch
+  changed_when: true
+
+- name: Reconnect through the new listener
+  ansible.builtin.meta: reset_connection
+
+- name: Read the sshd listeners after reconnecting
+  ansible.builtin.command: ss -Htln 'sport = :22'
+  changed_when: false
+  register: ssh_mesh_listeners
@@ -0,0 +1,9 @@
+#!/bin/sh
+# {{ ansible_managed }}
+# Restores the distro listener on all addresses. Armed as a timer by every
+# switchover; the play cancels it after a login through the new listener.
+systemctl stop {{ ssh_mesh_service }} {{ ssh_mesh_socket }}
+rm -f {{ ssh_mesh_udev_rule }} {{ ssh_mesh_socket_file }}{{ (' ' ~ ssh_mesh_service_dropin) if ssh_mesh_service_dropin else '' }}
+udevadm control --reload
+systemctl daemon-reload
+systemctl enable --now {{ ssh_mesh_distro_listener }}
@@ -0,0 +1,11 @@
+#!/bin/sh
+# {{ ansible_managed }}
+# Replaces the distro listener with the {{ ssh_mesh_interface }}-only socket.
+# Open sessions survive: they run in logind scopes and the sshd units use
+# KillMode=process. Any failure restores the distro listener at once.
+set -eu
+trap '/usr/local/sbin/ssh-mesh-revert' EXIT
+systemctl disable {{ ssh_mesh_disable_units }}
+systemctl stop {{ ssh_mesh_service }} {{ ssh_mesh_socket }}
+systemctl start {{ ssh_mesh_socket }}
+trap - EXIT
@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+# sshd holds the socket's listener; it stops with the socket, so it never keeps
+# a listener on a removed {{ ssh_mesh_interface }}. Replaces the Requires= that
+# enabling {{ ssh_mesh_socket }} would add.
+[Unit]
+BindsTo={{ ssh_mesh_socket }}
+After={{ ssh_mesh_socket }}
@@ -0,0 +1,8 @@
+# {{ ansible_managed }}
+# Listen on {{ ssh_mesh_interface }} only. The unit is disabled; the udev rule
+# starts it whenever {{ ssh_mesh_interface }} appears.
+[Socket]
+ListenStream=
+ListenStream=22
+BindToDevice={{ ssh_mesh_interface }}
+BindIPv6Only=both
@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+# Started by the udev rule whenever {{ ssh_mesh_interface }} appears; BindToDevice
+# stops it when the interface goes away. No [Install]: waiting for the device
+# from sockets.target would hold up boot until NetBird is up.
+[Unit]
+Description=OpenSSH on {{ ssh_mesh_interface }} only
+
+[Socket]
+ListenStream=22
+BindToDevice={{ ssh_mesh_interface }}
+BindIPv6Only=both
+Accept=yes
@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+# NetBird deletes and recreates {{ ssh_mesh_interface }} on every engine restart.
+SUBSYSTEM=="net", KERNEL=="{{ ssh_mesh_interface }}", ENV{SYSTEMD_WANTS}+="{{ ssh_mesh_socket }}"
@@ -0,0 +1,10 @@
+---
+# No packaged socket; sshd.socket (Accept=yes) pairs with the packaged
+# per-connection sshd@.service, since sshd -D cannot take a listening fd.
+ssh_mesh_socket: sshd.socket
+ssh_mesh_service: sshd.service
+ssh_mesh_distro_listener: sshd.service
+ssh_mesh_disable_units: sshd.service
+ssh_mesh_socket_file: /etc/systemd/system/sshd.socket
+ssh_mesh_socket_template: sshd.socket.j2
+ssh_mesh_service_dropin: ""
@@ -0,0 +1,10 @@
+---
+# Ubuntu's ssh.socket (Accept=no) hands its listener to ssh.service. The
+# drop-in is named like sshd-socket-generator's output so it masks it.
+ssh_mesh_socket: ssh.socket
+ssh_mesh_service: ssh.service
+ssh_mesh_distro_listener: ssh.socket
+ssh_mesh_disable_units: ssh.service ssh.socket
+ssh_mesh_socket_file: /etc/systemd/system/ssh.socket.d/addresses.conf
+ssh_mesh_socket_template: ssh.socket-addresses.conf.j2
+ssh_mesh_service_dropin: /etc/systemd/system/ssh.service.d/ssh-mesh-only.conf
@@ -12,7 +12,10 @@ The image is the official archiso **releng** profile plus:
 - `edge-install` in the live environment — partitions the NVMe
  (ESP + ext4 root, systemd-boot), pacstraps the base system incl.
  docker/python, configures DHCP via systemd-networkd, enables sshd and
-  drops the common homelab SSH keys for root (key-only, no root password)
+  sets up root login (key-only, no root password) with owner certificates
+  from the OpenBao SSH CA, fetched at build time from the `ssh-ca` role's
+  URL, and the break-glass key (YubiKey NEO, PIV) as fallback:
+  `ssh -I /usr/lib/opensc-pkcs11.so root@<host>`
 - the extra disk tools the installer needs

 Division of labour: the ISO produces an **ansible-ready base system**;
@@ -30,7 +33,8 @@ sudo ./build.sh          # ISO lands in ./out/

 The profile is assembled at build time from `/usr/share/archiso/configs/releng`
 plus the overlay in `airootfs/` — the repo carries only the delta, so
-archiso updates flow in automatically.
+archiso updates flow in automatically. The build needs openbao.renner.uno
+for the CA key; rebuild the ISO after a CA rotation.

 ## Install

@@ -38,8 +42,7 @@ archiso updates flow in automatically.
 2. Boot the target from it (UEFI, Secure Boot off; one-time F12/setup
   visit if the box doesn't already boot USB first)
 3. Either on the console (auto-login as root) or headless —
-   `ssh root@<dhcp-ip>` works out of the box, the live env carries the
-   homelab keys — run `edge-install`. It auto-detects the internal disk
+   `ssh root@<dhcp-ip>` with an owner certificate — run `edge-install`. It auto-detects the internal disk
   (NVMe, else eMMC — covers the Wyse 3040 — else first non-USB disk),
   shows the target and requires typing the hostname before wiping
   anything. Override with `edge-install /dev/sdY`.
@@ -1,4 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348
@@ -1,4 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348
@@ -81,6 +81,17 @@ ln -sf /run/systemd/resolve/stub-resolv.conf /mnt/etc/resolv.conf
 install -d -m700 /mnt/root/.ssh
 install -m600 /root/edge-authorized_keys /mnt/root/.ssh/authorized_keys

+# SSH user CA trust as baked into this ISO (owner certificates for root).
+install -m644 -t /mnt/etc/ssh /etc/ssh/trusted_user_ca_keys /etc/ssh/revoked_keys
+install -d -m755 /mnt/etc/ssh/auth_principals /mnt/etc/ssh/sshd_config.d
+install -m644 /etc/ssh/auth_principals/root /mnt/etc/ssh/auth_principals/root
+install -m644 /etc/ssh/sshd_config.d/50-ssh-ca.conf /mnt/etc/ssh/sshd_config.d/
+# sshd -t/-T need host keys; sshdgenkeys would create them on first boot.
+ssh-keygen -A -f /mnt
+arch-chroot /mnt sshd -t
+arch-chroot /mnt sshd -T | grep -qix 'trustedusercakeys /etc/ssh/trusted_user_ca_keys' \
+  || { echo "installed sshd does not use the SSH CA"; exit 1; }
+
 arch-chroot /mnt systemctl enable \
  systemd-networkd systemd-resolved systemd-timesyncd sshd docker

@@ -10,6 +10,23 @@ profile=/tmp/profile
 cp -r /usr/share/archiso/configs/releng/. "$profile"
 cp -r /src/airootfs/. "$profile/airootfs/"

+# SSH user CA trust from the ssh-ca role (mounted at /ssh-ca): the live system
+# and the installed one accept owner certificates for root, like the managed
+# hosts. The CA public key is fetched from OpenBao at build time.
+ca_url=$(sed -n 's/^ssh_ca_public_key_url: "\(.*\)"$/\1/p' /ssh-ca/defaults/main.yml)
+principal=$(sed -n 's/^ssh_ca_owner_principal: \(.*\)$/\1/p' /ssh-ca/defaults/main.yml)
+[ -n "$ca_url" ] && [ -n "$principal" ] || { echo "ssh-ca defaults not readable"; exit 1; }
+ca_key=$(curl -sSf "$ca_url")
+[[ "$ca_key" == "ssh-ed25519 AAAA"* ]] || { echo "unexpected CA key from $ca_url"; exit 1; }
+sshd_conf=$(grep -vx '# {{ ansible_managed }}' /ssh-ca/templates/ssh-ca.conf.j2)
+! grep -q '{[{%]' <<<"$sshd_conf" || { echo "ssh-ca.conf.j2 has template logic"; exit 1; }
+ssh_dir="$profile/airootfs/etc/ssh"
+install -d "$ssh_dir/sshd_config.d" "$ssh_dir/auth_principals"
+printf '%s\n' "$ca_key" > "$ssh_dir/trusted_user_ca_keys"
+cp /ssh-ca/files/revoked_keys "$ssh_dir/revoked_keys"
+printf '%s\n' "$principal" > "$ssh_dir/auth_principals/root"
+printf '%s\n' "$sshd_conf" > "$ssh_dir/sshd_config.d/50-ssh-ca.conf"
+
 # Tools the installer script needs in the live environment (idempotent —
 # skip entries releng already ships).
 for pkg in gptfdisk dosfstools e2fsprogs; do
@@ -13,6 +13,7 @@ mkdir -p "$out"

 docker run --rm --privileged \
  -v "$here:/src:ro" \
+  -v "$here/../../ansible/roles/ssh-ca:/ssh-ca:ro" \
  -v "$out:/out" \
  archlinux:latest /src/build-inner.sh

@@ -52,9 +52,9 @@ kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste key

 **Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
 the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
-from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
-itself. The shamir key above became the **recovery key** in the migration; it
-is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
+from `~/.config/homelab-secrets/seal-key.age`) and every restarted replica
+unseals itself. The shamir key above became the **recovery key** in the
+migration; it is `~/.config/homelab-secrets/openbao-recovery-key.age`. The server
 runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
 one is neither a voter nor a standby, so a replica that stays sealed after a
 restart — wrong or missing seal-key Secret — costs redundancy, two cost
@@ -237,19 +237,21 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
 - **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
  dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
  `openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
-  A raft snapshot restore needs the matching seal key — which lives in
-  `pass`, outside the cluster (GPG key backed up externally 2026-07-14).
+  A raft snapshot restore needs the matching seal key — which lives outside
+  the cluster in `~/.config/homelab-secrets/seal-key.age` (age, encrypted to
+  both YubiKeys and the offline paper key; copy on nas-2).
  A failed snapshot Job is the sealed-after-restart alarm. Still owed:
  one tested restore before phase 4.
- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
+- **Cluster rebuild order:** seal-key Secret from `seal-key.age` → Flux brings up
  openbao (empty) → init → restore raft snapshot → replicas unseal themselves
  → ClusterSecretStore Ready → ESO materializes Secrets → apps start
  (`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
  hand). Until then, secret-consuming apps simply stay Pending on missing
  Secrets — same as any missing dependency.
- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
-  offline-capable). With those, everything is recoverable from a raft
-  snapshot alone.
+- **Break-glass:** `~/.config/homelab-secrets` holds the seal key and
+  recovery key (YubiKeys + paper key, copy on nas-2);
+  `~/.config/openbao/root-token.age` holds the root token (YubiKeys). With
+  those, everything is recoverable from a raft snapshot alone.
 - **Ansible Vault password:** `ansible/vault-password.sh` reads
  `ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
  2026-07-14. It gates decryption of several Ansible-Vault files (PBS
@@ -67,17 +67,24 @@ These must exist outside the cluster or the rebuild cannot complete:
      `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
      files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
      `MIGRATION-sops-openbao.md`.)
- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
-      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
-      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
-      and the GPG private key backed up off-machine. The seal key is the one
-      the snapshots are encrypted under: without it, no OpenBao snapshot can be
-      restored and no secret-consuming app can start.
+- [ ] **OpenBao seal key, recovery key, restore + pull credentials** — age
+      files in `~/.config/homelab-secrets/` (`seal-key.age`,
+      `openbao-recovery-key.age` — the former shamir key —, `s3-access.age`,
+      `s3-secret.age`, `ghcr-pull.age`, `hetzner-cloud-api-secret.age`),
+      encrypted to both YubiKeys and the
+      offline paper key in the safe; a copy is on nas-2 in
+      `~ladmin/homelab-secrets-backup` (`~/.ssh/ssh-keys-rotate dr-copy`). The
+      root token is `~/.config/openbao/root-token.age` (YubiKeys only). Needed
+      to decrypt: one YubiKey with its PIV PIN, or the paper key. The seal key
+      is the one the snapshots are encrypted under: without it, no OpenBao
+      snapshot can be restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
      (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
      `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
      copy is the one that survives.
- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, plus the
+- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, `age` +
+      `age-plugin-yubikey` (the plugin is only needed for the YubiKeys; the
+      paper key needs `age` alone), plus the
      OpenTofu backend + UniFi credentials the `opentofu/` stack needs. No
      Proxmox credentials — there is no Proxmox any more.
 - [ ] **Install media** written and to hand: the Image Factory ISO named in
@@ -299,14 +306,16 @@ output** — a green step over a still-forming Ceph is a known trap.
 ## Step 4 — openbao: restore the snapshot

 `bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
-`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
+`openbao-seal-key` Secret from `~/.config/homelab-secrets/seal-key.age` before Flux brings OpenBao up, initialises
 the empty server, restores the newest raft snapshot straight out of the Velero
 kopia repository, and waits until every replica has unsealed itself from the
 static seal. What follows is the same procedure by hand, for when the script
 is not an option.

 OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
-`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
+`key`, the 32-byte base64 key in `~/.config/homelab-secrets/seal-key.age`,
+age-encrypted to the owner's YubiKeys and the offline paper key; a copy sits on
+nas-2 in `~ladmin/homelab-secrets-backup`). The
 snapshots are encrypted under that key, so it must be the same key the backed
 up cluster used — a different one restores nothing. Until the snapshot is
 restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
@@ -316,7 +325,8 @@ Secrets, and every secret-consuming app stays Pending.
 # 1. The seal key, before the pods start (a pod without it never starts).
 kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n openbao create secret generic openbao-seal-key \
-  --from-literal=key="$(pass show homelab/openbao/seal-key)"
+  --from-literal=key="$(age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age)"
+# Both YubiKeys lost: -i <file holding the paper key> instead.

 # 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
 #    initialise it. With an auto-seal, init hands out recovery keys and the
@@ -337,7 +347,8 @@ kubectl -n openbao exec -it openbao-0 -- \

 # 4. The server seals against the snapshot's keyring and unseals itself from
 #    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
-#    to type. Afterwards the tokens and recovery key from pass apply again.
+#    to type. Afterwards the operator's root token
+#    (~/.config/openbao/root-token.age) and recovery key apply again.
 kubectl -n openbao get pods -w
 ```

@@ -352,7 +363,7 @@ kubectl -n <ns> get externalsecret                # want: SecretSynced
 > A replica that stays **sealed** after a restart could not use the static
 > seal — the Secret is missing or does not match the key the data was
 > written under. Fix the Secret and restart the pod; `bao operator unseal`
-> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
+> with the recovery key (`~/.config/homelab-secrets/openbao-recovery-key.age`) does not
 > unseal an auto-sealed server and is only for recovery-mode operations.
 > Details + rationale: `MIGRATION-sops-openbao.md`.

@@ -434,7 +445,7 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
  Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
  Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
  Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
-  other secrets are read from `pass` or streamed in at run time and never live
+  other secrets are decrypted from `~/.config/homelab-secrets` or streamed in at run time and never live
  in CI.
 - **OpenBao unseals itself, but only under the seal key the snapshot was
  written with.** Three raft replicas mean the service survives losing one; a
@@ -16,6 +16,8 @@ resources:
  - networkpolicy.yaml
  # Nightly prune of old CI image builds (REN-1203).
  - package-retention.yaml
+  # OpenBao SSH CA public key for certificate logins.
+  - ssh-ca.yaml
  - kumamonitors.yaml
 configMapGenerator:
  - name: gitea-package-retention
@@ -0,0 +1,37 @@
+# Public key of the OpenBao SSH user CA (ssh-client-signer), for Gitea's
+# built-in SSH server to accept owner and claude-bot certificates. OpenBao
+# serves it unauthenticated; ESO reads it from the in-cluster Service, so the
+# key has one source. Gitea reads it from env at start; Reloader restarts the
+# StatefulSet when the Secret changes.
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: openbao-ssh-ca
+  namespace: gitea
+spec:
+  provider:
+    webhook:
+      url: http://openbao.openbao.svc.cluster.local:8200/v1/ssh-client-signer/public_key
+      method: GET
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-ssh-ca
+  namespace: gitea
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: openbao-ssh-ca
+  target:
+    name: gitea-ssh-ca
+    # OpenBao ends the body with a newline; it must not reach app.ini.
+    template:
+      engineVersion: v2
+      data:
+        trusted-user-ca-keys: "{{ .ca | trim }}"
+  data:
+    - secretKey: ca
+      remoteRef:
+        key: public_key
@@ -5,6 +5,9 @@ metadata:
  namespace: gitea
  labels:
    app.kubernetes.io/name: gitea
+  annotations:
+    # A CA rotation changes gitea-ssh-ca; Gitea reads it only at start.
+    secret.reloader.stakater.com/reload: gitea-ssh-ca
 spec:
  serviceName: gitea
  replicas: 1 # SQLite => single writer. Do NOT scale up without moving to Postgres first.
@@ -85,6 +88,16 @@ spec:
            # HAS CHANGED". Paths are relative to APP_DATA_PATH (/var/lib/gitea/data).
            - name: GITEA__server__SSH_SERVER_HOST_KEYS
              value: "ssh/ssh_host_ed25519_key,ssh/ssh_host_rsa_key"
+            # OpenBao SSH CA (ssh-ca.yaml): certificates log in as the user whose
+            # name is a principal of the cert. Each user registers that
+            # principal once (Settings > SSH / GPG Keys > Manage principals).
+            - name: GITEA__server__SSH_TRUSTED_USER_CA_KEYS
+              valueFrom:
+                secretKeyRef:
+                  name: gitea-ssh-ca
+                  key: trusted-user-ca-keys
+            - name: GITEA__server__SSH_AUTHORIZED_PRINCIPALS_ALLOW
+              value: username
            # Gitea Actions was decommissioned — keep it off.
            - name: GITEA__actions__ENABLED
              value: "false"
@@ -49,7 +49,7 @@ spec:
              failure would cost quorum. Check `kubectl -n openbao get pods`,
              the pod log and `bao status` in the not-ready pod. If it is
              sealed, the static seal failed: verify the openbao-seal-key
-              Secret matches `pass show homelab/openbao/seal-key`.
+              Secret matches ~/.config/homelab-secrets/seal-key.age (age, YubiKey).

        - alert: OpenBaoQuorumLost
          # Raft quorum of a three-node cluster is two. At one ready replica
@@ -76,7 +76,7 @@ spec:
              surface only when something rotates or a new secret is needed.
              Replicas unseal themselves from the static seal; one that stays
              sealed could not. Check its log and the openbao-seal-key Secret
-              against `pass show homelab/openbao/seal-key`, fix the Secret and
+              against ~/.config/homelab-secrets/seal-key.age, fix the Secret and
              restart the pod. The recovery key cannot unseal an auto-sealed
              server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.

@@ -22,7 +22,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: gotenberg
-          image: gotenberg/gotenberg:8.38.1 # {"$imagepolicy": "flux-system:gotenberg"}
+          image: gotenberg/gotenberg:8.39.0 # {"$imagepolicy": "flux-system:gotenberg"}
          args:
            - "gotenberg"
            - "--chromium-disable-javascript=true"
@@ -53,7 +53,7 @@ spec:
          type: RuntimeDefault
      containers:
        - name: wiki
-          image: gitea.renner.uno/konairius/wiki:build.1793086240 # {"$imagepolicy": "flux-system:wiki"}
+          image: gitea.renner.uno/konairius/wiki:build.1793094262 # {"$imagepolicy": "flux-system:wiki"}
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
@@ -35,7 +35,8 @@ set -euo pipefail
 # The mirror is kept current server-side by Gitea's native push-mirror. Full
 # cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
 #
-# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt
+# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt,
+# age + age-plugin-yubikey (the DR secrets in ~/.config/homelab-secrets)

 TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -190,13 +191,16 @@ if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wai
 fi
 OPENBAO_NS="openbao"

-# Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be:
-# the pass store opens everything, and it stays on the operator's machine. Set
-# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
+# Where the secrets come from. On a workstation, age files in
+# HOMELAB_SECRETS_DIR, encrypted to the operator's YubiKeys and an offline paper
+# key (a copy sits on nas-2). HOMELAB_AGE_IDENTITY picks what decrypts them: the
+# YubiKey identity stubs by default (touch per file), the paper key's file when
+# both sticks are gone. In the in-cluster Job there is neither — and there
+# should not be: those keys open everything, and they stay with the operator.
+# Set OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
 # operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/seal-key \
+#     age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age \
 #       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
@@ -240,6 +244,15 @@ git_token_for_source() {
    fi
 }

+# Literal ~ on purpose: the commands are eval'd on the workstation, which
+# expands it there, and the in-cluster Job prints them for the operator to run
+# on that workstation, where the Job's own HOME (/work) would be wrong.
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_SECRETS_DIR:-}" ]] || HOMELAB_SECRETS_DIR='~/.config/homelab-secrets'
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_AGE_IDENTITY:-}" ]] || HOMELAB_AGE_IDENTITY='~/.config/age/yubikeys.txt'
+AGE_SECRET="age -d -i $HOMELAB_AGE_IDENTITY $HOMELAB_SECRETS_DIR"
+
 # Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
 OPENBAO_REQUIRED_SECRETS=(
    "sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
@@ -255,18 +268,18 @@ OPENBAO_REQUIRED_SECRETS=(
    # Emptiness is not tested here on purpose — require_secrets rejects an empty
    # file and names what is missing.
    "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
-    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
-    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
+    "s3-access|Hetzner restore access key|$AGE_SECRET/s3-access.age"
+    "s3-secret|Hetzner restore secret key|$AGE_SECRET/s3-secret.age"
    # The static seal key. The snapshots are encrypted under it, so it has to
    # be the one the backed-up cluster ran with — a fresh key restores nothing.
-    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
+    "seal-key|OpenBao seal key|$AGE_SECRET/seal-key.age"
    # Read by the node registry auth (see registry_auth_patch): the images an app
    # needs while the cluster is still being built are mirrored to a private
    # ghcr.io namespace, and a node has to be able to pull them before anything
    # in the cluster exists to hold a credential. Demanded up front like the
    # rest, because a rebuild that discovers it missing has already wiped the
    # disks.
-    "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
+    "ghcr-pull|GHCR pull token for the image mirror|$AGE_SECRET/ghcr-pull.age"
 )
 # Asked for by name only, never by a bare require_secrets: a rebuild of the
 # real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
@@ -281,8 +294,8 @@ HCLOUD_REQUIRED_SECRETS=(
 # during a real recovery or in the in-cluster Job.
 #
 # The only difference between those two is where the material comes from, and it
-# is confined to this function: on a laptop the commands in the list produce it
-# from pass; in the Job the operator streams it in. Everything downstream cannot
+# is confined to this function: on a laptop the commands in the list decrypt
+# it from HOMELAB_SECRETS_DIR; in the Job the operator streams it in. Everything downstream cannot
 # tell the difference — which is the point, because the recovery path must not be
 # a path that only gets exercised during a recovery.
 # require_secrets [name...]
@@ -311,7 +324,7 @@ require_secrets() {
        wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
    fi

-    # Which branch: fetch from pass, or wait for someone to stream files in.
+    # Which branch: decrypt locally, or wait for someone to stream files in.
    #
    # The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
    # FUNCTION sets it. Once a subset call had run — hcloudtest asks for
@@ -322,7 +335,7 @@ require_secrets() {
    # maintenance.
    #
    # SECRETS_SELF_FETCH records which world we are in, and it is decided once:
-    # an empty OPENBAO_SECRETS_DIR on entry means pass, an inherited one means
+    # an empty OPENBAO_SECRETS_DIR on entry means local age decryption, an inherited one means
    # the Job. Later calls follow that decision instead of re-deriving it from
    # state this function has since changed.
    if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
@@ -342,12 +355,14 @@ require_secrets() {
            name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
            # Already collected by an earlier call — a subset call fetches only
            # what it asked for, so the full call that follows must fill the
-            # gaps rather than re-run every command. `pass show` on a locked
-            # store prompts, and prompting twice for the same key is how an
+            # gaps rather than re-run every command. Each decryption waits for
+            # a YubiKey touch, and asking twice for the same key is how an
            # unattended run turns into a hung one.
            [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
-            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
-                echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
+            # stderr stays on the terminal: age-plugin-yubikey asks for the
+            # PIN and the touch there, and a captured prompt is a silent hang.
+            echo
+            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name"; then
                fail "could not obtain the $what with: $how"
            fi
            [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
@@ -382,7 +397,7 @@ OPENBAO_NS_HINT=vmtest

 # Reads one secret by its name in the list above. In the Job that is the file the
 # operator streamed in; on a workstation it runs the command the list names,
-# which for these is `pass show <path>`.
+# which for these is `age -d` of a file in HOMELAB_SECRETS_DIR.
 #
 # The list is the only place the paths appear. They used to sit here as
 # constants as well, which is how the same value ends up defined twice and one
@@ -1486,7 +1501,8 @@ fi

 # Before anything destructive, and after the vmtest block has decided which
 # phases will run. A rebuild that will need these has no business wiping disks
-# first, only to discover an hour later that pass is locked or a token is unset —
+# first, only to discover an hour later that no YubiKey is plugged in or a
+# token is unset —
 # by then the old cluster is gone and the new one cannot be finished.
 #
 # In the Job this is where the file-waiting branch blocks, which is the right
@@ -1994,7 +2010,8 @@ bao_exec() {
 # have been doing this all along.
 #
 # Reading them needs three things, all of which live OUTSIDE the cluster, which
-# is the point: bucket credentials from pass (their own key, not Velero's),
+# is the point: bucket credentials from HOMELAB_SECRETS_DIR (their own key, not
+# Velero's),
 # kopia, and the repository password. The last is not a secret — Velero
 # hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
 # the offsite backup's confidentiality rests on bucket access alone.
@@ -2071,7 +2088,8 @@ restore_openbao() {
    # rather than an unseal key and the server unseals itself; the only thing
    # used from its output is the root token, for the next few seconds, and
    # then it stops existing: restoring replaces the keyring with the
-    # snapshot's, after which the root token and recovery key from pass apply.
+    # snapshot's, after which the operator's root token
+    # (~/.config/openbao/root-token.age) and recovery key apply.
    #
    # No -force. Without it OpenBao checks that the snapshot was written under
    # the seal key this server runs with, and refuses a mismatch by name
@@ -2355,7 +2373,7 @@ if phase_runs flux; then
    # Secret would leave every replica sealed for good at its next restart.
    # That is a migration, not a bootstrap step.
    #
-    # --from-literal, not --from-file: `pass show` ends its output with a
+    # --from-literal, not --from-file: the stored value ends with a
    # newline, the substitution strips it, and OpenBao rejects a key that is
    # not exactly 32 bytes of base64. The key sits in this process's argument
    # list for the length of the call, as it did in the pod's before.
@@ -80,6 +80,25 @@ spec:
          # community blocklists.
          - 162.55.50.97/32
          - 2a01:4f8:c2c:4579::1/128
+      postoverflows:
+        s01-whitelist:
+          # http-crawl-non_statics counts requests regardless of outcome, so a
+          # signed-in user of any image-heavy UI (Immich timeline, Music
+          # Assistant covers) trips it with a burst of 200s. Scanners trip it
+          # with 404s for paths that do not exist. Drop the overflow unless at
+          # least half of its requests were client errors (the alert carries
+          # the scenario's last 5 events, cache_size: 5). Only this scenario
+          # is affected; probing/sensitive-files/cve still fire on their own.
+          crawl-non-statics-success-whitelist.yaml: |
+            name: krenner/crawl-non-statics-success-whitelist
+            description: "Ignore http-crawl-non_statics bursts that mostly succeeded"
+            whitelist:
+              reason: "crawl burst of successful requests is a UI, not a scanner"
+              expression:
+                - >-
+                  evt.Overflow.Alert.GetScenario() == 'crowdsecurity/http-crawl-non_statics' &&
+                  count(evt.Overflow.Alert.Events, {.GetMeta('http_status') startsWith '4'}) * 2
+                  < len(evt.Overflow.Alert.Events)

    # mTLS between agents and the LAPI. Do not switch back to password auth:
    # the LAPI runs without persistence, so its machine-registration DB is
@@ -1,6 +1,9 @@
 # OpenBao policy: ssh-sign-owner
 # Sign the owner's SSH public keys on ssh-client-signer role owner. Bound to auth/oidc role owner. NOT applied by Flux.
 #
+#   bao secrets enable -path=ssh-client-signer -max-lease-ttl=4380h ssh   # 768h default would cap the 6-month certs
+#   bao write ssh-client-signer/config/ca generate_signing_key=true key_type=ed25519
+#   bao write ssh-client-signer/roles/owner @kubernetes/infrastructure/openbao/ssh-ca/role-owner.json
 #   bao policy write ssh-sign-owner kubernetes/infrastructure/openbao/policies/ssh-sign-owner.hcl
 #   bao auth enable oidc
 #   bao write auth/oidc/config oidc_discovery_url=https://auth.renner.uno/application/o/openbao/ \
@@ -1,9 +0,0 @@
-locals {
-  common_ssh_keys = join("\n", [
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab",
-    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310"
-  ])
-
-}
@@ -43,12 +43,6 @@ provider "proxmox" {
  endpoint  = var.proxmox_api_url
  api_token = "${var.proxmox_user}!${var.proxmox_token_id}=${var.proxmox_token_secret}"
  insecure  = var.proxmox_tls_insecure
-
-  ssh {
-    agent       = false
-    username    = "root"
-    private_key = var.ssh_private_key
-  }
 }

 provider "unifi" {
@@ -28,16 +28,4 @@ variable "proxmox_tls_insecure" {
  default     = false
 }

-variable "ssh_public_key" {
-  description = "Public SSH key to inject into containers"
-  type        = string
-  default     = null
-}
-
-variable "ssh_private_key" {
-  description = "SSH private key for Proxmox provider to import disk images"
-  type        = string
-  sensitive   = true
-  default     = ""
-}