fix(secret-rotator): never fail the HA run after the store; stage the image rollout
Folds the good parts of #1412 into this PR and addresses both review-bot reviews (#1412 review 1789, #1414 review 1791): - Everything after the OpenBao store is best-effort. Decode, reconnect, listing and revokes run in reap_superseded(); reap_summary() turns any error into "reap skipped" plus a warning, so rotate_ha_token returns Ok. A failed run would start a second pod (backoffLimit: 1) that keeps the token just stored and its successor and reaps the token consumers still hold until ESO syncs, i.e. the #1081 outage. Two unit tests cover it. - Warn when the replaced token is not named "<client_name> [<6 hex>]": no later run reaps it, so it needs a manual revoke (from #1412). - Staged image rollout (from #1412): only the three HA CronJobs carry the flux-system:secret-rotator marker; the other 17 rotators stay on 2026-07-30 until a manual HA run validates the new binary, then a follow-up adds the marker to them. rotation/kustomization.yaml records the exception. - Merged current main (a6f37e2b,1691a0ac): the Reloader annotations for zone-configurator and speech-to-phrase are already there via #1409.
This commit is contained in:
@@ -45,7 +45,7 @@ existing `flux-system` GitRepository whose Secret was created by
|
||||
| wyoming-{whisper,openwakeword,speech-to-phrase} | `3.x` / `2.x` / `1.x` | |
|
||||
| authentik-gate-sync | `YYYY-MM-DD-HHMM` | In-house (build-ci-images.yml); newest tag wins |
|
||||
| kuma-sync | `YYYY-MM-DD-HHMM` | In-house (build-ci-images.yml); newest tag wins |
|
||||
| secret-rotator | `YYYY-MM-DD` | In-house (build-ci-images.yml); newest tag wins; one marker per rotator CronJob, all move together |
|
||||
| secret-rotator | `YYYY-MM-DD` | In-house (build-ci-images.yml); newest tag wins; staged: only the Home Assistant rotation CronJobs carry the marker so far |
|
||||
| gatus | `5.x` | v-prefix filter |
|
||||
| velero-plugin-for-aws | `1.13.x` | Tight: must match velero server major.minor |
|
||||
| gotenberg | `8.x` | Plain-variant filter (rejects `-libreoffice`/`-chromium`/`-cloudrun`/`-aws-lambda`) |
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -78,7 +78,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: sync-woodpecker
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["woodpecker-secret"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["cloudflare-roll"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -56,7 +56,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["cloudflare-roll"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -59,7 +59,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -57,7 +57,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate-mint
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
# the tag is free. Make-before-break: read -> mint -> store -> reap; a failed
|
||||
# store leaves the old token valid and stored (a no-op, not an outage). The
|
||||
# reap keeps the new AND the replaced token, because consumers read the k8s
|
||||
# Secret that ESO syncs only every 1h; the replaced one goes next quarter.
|
||||
# Secret that ESO syncs only every 1h; the replaced one goes next quarter. Reap
|
||||
# errors only warn: a Job retry would revoke the token consumers still hold.
|
||||
#
|
||||
# Reachability: HA :8123 (HA itself, not the :8124 XFF-anchor proxy) via the
|
||||
# home-assistant-bgp Service ClusterIP. Needs BOTH netpol halves —
|
||||
|
||||
@@ -14,6 +14,7 @@ kind: Kustomization
|
||||
#
|
||||
# Keep every rotator on the same tag: it is one binary containing all modes, so
|
||||
# splitting only makes the next bump partial.
|
||||
# Exception while staged: only home-assistant.yaml carries the image marker.
|
||||
#
|
||||
# ADDING A ROTATOR: copy an existing file, and check you did not inherit a
|
||||
# digest pin from an older example — nothing enforces this automatically, so it
|
||||
@@ -30,8 +31,8 @@ kind: Kustomization
|
||||
#
|
||||
# BUMPING the tag: Flux image automation (image-automation/secret-rotator.yaml)
|
||||
# moves every `flux-system:secret-rotator` marker to the newest dated tag the
|
||||
# registry lists, so only tags that were actually built are ever written. Every
|
||||
# rotator image line must carry that marker. To check a tag resolves by hand:
|
||||
# registry lists, so only tags that were actually built are ever written. Lines
|
||||
# without the marker are bumped by hand; verify the tag exists first:
|
||||
#
|
||||
# kubectl run tagcheck --rm -i --restart=Never \
|
||||
# --image=gitea.renner.uno/gitea_admin/secret-rotator:<DATE> \
|
||||
|
||||
@@ -93,7 +93,7 @@ spec:
|
||||
# it pulls, `random` is among its modes, and it carries the
|
||||
# RANDOM_BYTES>=16 floor documented below (RANDOM_BYTES=8 exits
|
||||
# "refusing to generate a weak secret").
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -59,7 +59,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["netbird-key"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -60,7 +60,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: rotate-mint
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -96,7 +96,7 @@ spec:
|
||||
memory: 48Mi
|
||||
containers:
|
||||
- name: sync-woodpecker
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["woodpecker-secret"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -82,7 +82,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
initContainers:
|
||||
- name: rotate-mint
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -129,7 +129,7 @@ spec:
|
||||
memory: 48Mi
|
||||
# Read-only review-bot PAT for the Anthropic backend on the Gitea MCP server.
|
||||
- name: rotate-mint-mcp
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["gitea-token"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
@@ -165,7 +165,7 @@ spec:
|
||||
memory: 48Mi
|
||||
containers:
|
||||
- name: sync-woodpecker
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["woodpecker-secret"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -63,7 +63,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rotate
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
|
||||
image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
|
||||
args: ["random"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
+78
-19
@@ -413,7 +413,7 @@ fn sync_woodpecker_secret() -> Result<String, Box<dyn Error>> {
|
||||
/// 2. mint the replacement
|
||||
/// 3. write it to OpenBao
|
||||
/// 4. only then revoke rotator-minted tokens other than the new and the
|
||||
/// outgoing one
|
||||
/// outgoing one (best-effort: an error here warns, never fails the run)
|
||||
///
|
||||
/// The outgoing token is NOT revoked in this run. Consumers read the k8s
|
||||
/// Secret, which ESO syncs from OpenBao only every refreshInterval, so revoking
|
||||
@@ -484,22 +484,49 @@ fn rotate_ha_token() -> Result<String, Box<dyn Error>> {
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Reap over a FRESH connection authenticated with the NEW token. HA binds
|
||||
// the WebSocket session to the refresh token behind its access token, so
|
||||
// the new token is the one session no revoke below can tear down.
|
||||
// From here on the rotation has succeeded: the new token is live and
|
||||
// stored, so nothing below may fail the run. A failed run makes the Job
|
||||
// start a second pod (backoffLimit), which would read the token just stored
|
||||
// as "current", keep it and its successor, and reap the token consumers
|
||||
// still hold until ESO syncs: the #1081 outage again. Reap errors only warn.
|
||||
if !ha::is_rotator_minted(&client_name, &old_name) {
|
||||
// An untagged token (pre-rotator original, or reseeded by hand) never
|
||||
// matches the reap, so no later run will revoke it.
|
||||
eprintln!(
|
||||
"warning: replaced token '{old_name}' ({old_id}) is not named '{client_name} [<6 hex>]', \
|
||||
so no later run will revoke it; revoke it by hand once consumers use '{unique_name}'"
|
||||
);
|
||||
}
|
||||
drop(ha);
|
||||
let fresh_id = ha::token_refresh_id(&fresh)?;
|
||||
let mut ha2 = ha::Ha::connect(&ha_url, &fresh)?;
|
||||
let reap = reap_superseded(&ha_url, &fresh, &client_name, &old_id);
|
||||
Ok(format!(
|
||||
"rotated {bao_path}#{field}: minted '{unique_name}', kept previous '{old_name}' ({old_id}) until the next rotation, {} — value not logged",
|
||||
reap_summary(reap)
|
||||
))
|
||||
}
|
||||
|
||||
// Keep the token just stored AND the one it replaced: the replaced token is
|
||||
// what every consumer's k8s Secret still holds until ESO syncs. Revoke only
|
||||
// rotator-minted tokens for this client_name other than those two — they
|
||||
// were superseded at least one full rotation ago.
|
||||
let targets = ha::reap_targets(&ha2.long_lived_tokens()?, &client_name, &[&fresh_id, &old_id]);
|
||||
/// Revoke rotator-minted tokens for `client_name` other than the one just
|
||||
/// stored and the one it replaced (`old_id`): those were superseded at least
|
||||
/// one full rotation ago. Returns (revoked, not revoked) as log labels.
|
||||
///
|
||||
/// Runs over a FRESH connection authenticated with the NEW token. HA binds the
|
||||
/// WebSocket session to the refresh token behind its access token, so deleting
|
||||
/// a token over its own session tears the session down before the delete-ack
|
||||
/// can be read (verified live). The new token is never a reap target, so its
|
||||
/// session survives every revoke below.
|
||||
fn reap_superseded(
|
||||
ha_url: &str,
|
||||
fresh: &str,
|
||||
client_name: &str,
|
||||
old_id: &str,
|
||||
) -> Result<(Vec<String>, Vec<String>), Box<dyn Error>> {
|
||||
let fresh_id = ha::token_refresh_id(fresh)?;
|
||||
let mut ha = ha::Ha::connect(ha_url, fresh)?;
|
||||
let targets = ha::reap_targets(&ha.long_lived_tokens()?, client_name, &[&fresh_id, old_id]);
|
||||
let mut reaped = Vec::new();
|
||||
let mut failed = Vec::new();
|
||||
for (id, name) in &targets {
|
||||
match ha2.revoke(id) {
|
||||
match ha.revoke(id) {
|
||||
Ok(()) => reaped.push(format!("'{name}' ({id})")),
|
||||
Err(e) => {
|
||||
eprintln!("warning: revoking superseded token '{name}' ({id}) failed and it MAY REMAIN VALID: {e}");
|
||||
@@ -507,13 +534,45 @@ fn rotate_ha_token() -> Result<String, Box<dyn Error>> {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((reaped, failed))
|
||||
}
|
||||
|
||||
/// Log fragment for the reap step. Infallible on purpose: a reap error after
|
||||
/// the store is reported as "reap skipped", never as a failed run.
|
||||
fn reap_summary(outcome: Result<(Vec<String>, Vec<String>), Box<dyn Error>>) -> String {
|
||||
match outcome {
|
||||
Ok((reaped, failed)) => {
|
||||
let reaped = if reaped.is_empty() { "none".to_string() } else { reaped.join(", ") };
|
||||
let failed = if failed.is_empty() {
|
||||
String::new()
|
||||
if failed.is_empty() {
|
||||
format!("revoked older: {reaped}")
|
||||
} else {
|
||||
format!("; NOT revoked (see warning): {}", failed.join(", "))
|
||||
};
|
||||
Ok(format!(
|
||||
"rotated {bao_path}#{field}: minted '{unique_name}', kept previous '{old_name}' ({old_id}) until the next rotation, revoked older: {reaped}{failed} — value not logged"
|
||||
))
|
||||
format!("revoked older: {reaped}; NOT revoked (see warning): {}", failed.join(", "))
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"warning: new token is live and stored, but reaping superseded tokens failed: {e}. \
|
||||
The run still succeeds (a Job retry would revoke the token consumers hold); \
|
||||
the next rotation reaps them."
|
||||
);
|
||||
"reap skipped (see warning)".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reap_error_after_store_is_not_a_failure() {
|
||||
assert_eq!(reap_summary(Err("connection refused".into())), "reap skipped (see warning)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reap_summary_lists_revoked_and_failed() {
|
||||
assert_eq!(reap_summary(Ok((vec![], vec![]))), "revoked older: none");
|
||||
let s = reap_summary(Ok((vec!["'a [111111]' (x)".into()], vec!["'a [222222]' (y)".into()])));
|
||||
assert_eq!(s, "revoked older: 'a [111111]' (x); NOT revoked (see warning): 'a [222222]' (y)");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user