diff --git a/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl b/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl index d153747e..64d25c50 100644 --- a/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl +++ b/kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl @@ -13,13 +13,32 @@ # bound_service_account_namespaces=openbao \ # policies=rotator-proxmox ttl=15m # -# BOOTSTRAP CREDENTIAL you must provision first (one-time, by hand): -# A SEPARATE privileged Proxmox API token (NOT the pve-exporter@pve!monitoring -# token this Job rotates) with permission to manage tokens on pve-exporter@pve -# (e.g. root@pam!, or a dedicated rotator token with the right ACL): +# BOOTSTRAP CREDENTIAL (PROVISIONED — recorded here for rebuild/DR). +# +# A SEPARATE least-privilege Proxmox identity, NOT root@pam. PVE has no +# per-user ACL path (/access/users/ is rejected — see AccessControl.pm +# check_path), and the token endpoints authorise via a `userid-group` check. +# So the tightest possible scope is a group containing only the target user: +# +# pveum role add SecretRotator -privs "User.Modify" +# pveum user add secret-rotator@pve +# pveum group add secret-rotated +# pveum user modify pve-exporter@pve --groups secret-rotated +# pveum acl modify /access/groups/secret-rotated \ +# --users secret-rotator@pve --roles SecretRotator +# pveum user token add secret-rotator@pve rotate --privsep 0 +# +# This grants exactly one thing: managing API tokens of users in the +# secret-rotated group. Verified: no VM/storage visibility, and minting a +# root@pam token returns 403. +# # bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \ -# api-url=https://jupiter.renner.uno:8006 \ -# auth='pve-rotator@pve!rotate=' +# api-url=https://pve.apps.svc.cluster.local:8006 \ +# auth='secret-rotator@pve!rotate=' +# +# api-url MUST be the in-cluster Service (backed by all three PVE nodes). +# The external name pve.renner.uno does NOT work from a pod — pods cannot +# hairpin to their own cluster's ingress LB and the Job hangs. # # Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy). diff --git a/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml b/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml index 84318611..80b24937 100644 --- a/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml +++ b/kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml @@ -7,17 +7,28 @@ # syncs the pve-exporter-config Secret (ns monitoring, 10m refresh); Reloader # restarts the pve-exporter Deployment to pick up the new config file. # -# ┌─ SUSPENDED (spec.suspend: true) — the riskiest Class-B job ──────────────┐ -# │ UNTESTED. Before enabling: │ -# │ 1. Provision the bootstrap privileged Proxmox token (see rotator-proxmox.hcl). │ -# │ 2. VERIFY the token's privsep setting. This script re-creates with │ -# │ privsep=0 (token inherits pve-exporter@pve's own perms → no separate │ -# │ token-ACL to re-apply). If the current monitoring token uses privsep=1 │ -# │ with its OWN ACL, you MUST re-grant that ACL after re-creation, or the │ -# │ exporter loses read access. Check: GET /access/users/pve-exporter@pve/token/monitoring │ -# │ 3. DELETE+create causes a brief window with NO valid token — pve-exporter │ -# │ metrics gap until ESO(≤10m)+Reloader restart. Quarterly/off-hours ok. │ -# │ 4. Test on-demand, confirm metrics resume, THEN set suspend: false. │ +# ┌─ TESTED end-to-end and ENABLED ──────────────────────────────────────────┐ +# │ The privsep caveat from #306 is RESOLVED: pve-exporter@pve!monitoring is │ +# │ privsep=0 and its ACL (/ → PVEAuditor) lives on the USER, not the token, │ +# │ so DELETE+re-create with privsep=0 loses no permission — nothing to │ +# │ re-grant. Verified by two full rotations; metrics resumed both times. │ +# │ │ +# │ Bootstrap credential is a dedicated least-privilege PVE identity, NOT │ +# │ root (see rotator-proxmox.hcl): secret-rotator@pve!rotate, holding role │ +# │ SecretRotator (User.Modify) scoped to /access/groups/secret-rotated — │ +# │ a group whose only member is pve-exporter@pve. Verified it can mint and │ +# │ delete that user's tokens, sees no VMs/storage, and is 403 on minting a │ +# │ root@pam token. │ +# │ │ +# │ api-url is the in-cluster Service https://pve.apps.svc.cluster.local:8006 │ +# │ (all three PVE nodes back it, so a single node being down is tolerated). │ +# │ Do NOT use the external name pve.renner.uno — pods cannot hairpin to │ +# │ their own cluster's ingress LB and the job hangs. Do NOT pin a node IP │ +# │ or a single node FQDN (jupiter/…): both are a needless SPOF. │ +# │ │ +# │ DELETE+create still leaves a brief window with no valid token — │ +# │ pve-exporter metrics gap until ESO(≤10m)+Reloader restart. Quarterly │ +# │ off-hours schedule keeps that acceptable. │ # └───────────────────────────────────────────────────────────────────────────┘ apiVersion: v1 kind: ServiceAccount @@ -39,7 +50,7 @@ metadata: app.kubernetes.io/name: pve-exporter app.kubernetes.io/component: secret-rotator spec: - suspend: true # UNTESTED + monitoring gap — see header; enable only after test + suspend: false # tested end-to-end (two full rotations, metrics resumed) — see header schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin timeZone: Europe/Berlin concurrencyPolicy: Forbid