homelab/kubernetes/vmtest-job.yaml

# Runs `bootstrap.sh vmtest` inside the cluster instead of on a workstation.
#
# NOT reconciled by Flux — deliberately outside kubernetes/clusters. This is a
# one-shot test, and a Kustomization would recreate it after every completion.
# It is applied by the vmtest pipeline, or by hand for a single run.
#
# Why in-cluster at all: the test wants ~30G for four VMs (one control plane at
# 6G, three OSD workers at 8G). The workstation has 31G with a desktop on it —
# a single run left under 1G free and two overlapping runs OOM-killed the
# session. The nodes have ~94G each with about 70G unused.
#
# The whole script runs here, flux and OpenBao included, so that what a rebuild
# rehearses is what a real recovery would execute. The OpenBao seal key is not
# baked in and not stored as a Secret here: the operator streams the secrets
# into an in-memory emptyDir at the moment the run asks for them, and they die
# with the Job without ever reaching this cluster's etcd or a backup of it.
apiVersion: v1
kind: Namespace
metadata:
  name: vmtest
  labels:
    # The pod is privileged; say so rather than have the admission policy
    # reject it for reasons that look unrelated.
    pod-security.kubernetes.io/enforce: privileged
---
apiVersion: batch/v1
kind: Job
metadata:
  name: vmtest
  namespace: vmtest
spec:
  backoffLimit: 0        # a failed rebuild test is a result, not something to retry
  ttlSecondsAfterFinished: 86400
  template:
    metadata:
      labels:
        app.kubernetes.io/name: vmtest
    spec:
      restartPolicy: Never
      # Nothing here should be able to reach the cluster it runs in. The phase
      # points KUBECONFIG at its own throwaway cluster, and without a mounted
      # token a stray kubectl cannot fall back to this one.
      automountServiceAccountToken: false
      containers:
        - name: vmtest
          # Pinned by digest, not :latest. Spegel mirrors mutable tags and keeps
          # serving what it cached: a run on k8s-3 pulled a ci-vmtest:latest
          # three builds old, which predated OVMF, and died with "the required
          # flash image was not found" before the first VM booted. The tag is
          # carried alongside for readability — containerd resolves the digest.
          image: gitea.renner.uno/gitea_admin/ci-vmtest:2026-09-03@sha256:a8145a76896ef9aa6fdb58dc96b9bdc22b53f4e55f04d9e6de289f35b75dbd81
          # Privileged for /dev/kvm and for the bridges and taps the provisioner
          # builds. It needs no hostNetwork: the provisioner's network lives in
          # this pod's own namespace, and the script that talks to 10.5.0.x runs
          # in the same pod.
          securityContext:
            privileged: true
          # A hard ceiling, because the failure mode is known: an unbounded run
          # is what OOM-killed the workstation. 40G is the VMs (16G control
          # plane + 3x8G workers), the rest is qemu overhead and the tooling.
          # The workers carry an OSD, a mon and a mgr each; at 4G they ran out
          # of room for the CNI DaemonSets themselves. The control plane went
          # 6G → 8G → 16G: at both smaller sizes it ran at 99% of allocatable
          # memory and the kubelet evicted kube-apiserver to admit a
          # node-critical pod — bootstrap.sh carries the full account next to
          # the sizing flags. The hosts have ~95GiB each, so this fits.
          resources:
            requests:
              # Less than the 14 guest vCPUs the four VMs declare, on purpose:
              # the guests are never all runnable at once (one cluster booting
              # in sequence), and there is no CPU limit, so the pod may burst
              # past the request when the node has headroom. The request only
              # keeps the scheduler from stacking this next to another big
              # burst consumer.
              cpu: "6"
              memory: 42Gi
            limits:
              memory: 48Gi
          env:
            # talosctl caches the image and the CNI bundle under $HOME; point it
            # at the scratch volume rather than the container layer.
            - name: HOME
              value: /work
            # The VM disks. Short on purpose — the provisioner puts a qemu
            # monitor socket under it and sun_path caps the path at 108 bytes.
            - name: VM_STATE
              value: /work/vm
            # Which branch to test. main unless a run is checking a change.
            - name: BRANCH
              value: main
            # Secrets are streamed in here at the moment the run needs them,
            # never baked into the image and never stored as a Secret. They live
            # in this emptyDir for the length of the Job and go with it — so
            # they do not reach etcd, or any backup of it.
            - name: OPENBAO_SECRETS_DIR
              value: /secrets
          command: ["/bin/bash", "-ec"]
          args:
            - |
              # From the GitHub mirror, not in-cluster Gitea. A rebuild test
              # that clones from the cluster it is rehearsing the loss of
              # cannot start when it matters — and a run already died here on
              # resolving gitea.renner.uno.
              git clone --depth 1 --branch "$BRANCH" \
                https://github.com/konairius/Homelab.git /work/homelab
              exec /work/homelab/kubernetes/bootstrap.sh vmtest
          volumeMounts:
            - name: work
              mountPath: /work
            - name: secrets
              mountPath: /secrets
            - name: dev-kvm
              mountPath: /dev/kvm
      volumes:
        # The VM disks and the provisioner state. emptyDir on the node's disk
        # rather than a PVC: it is scratch that dies with the Job, and putting it
        # on Ceph would have the test's own writes compete with the storage it is
        # meant to leave alone.
        - name: work
          emptyDir:
            # The VM images are sparse, so this bounds what is actually written,
            # not the 232GiB the geometry declares. 60Gi was already half spent
            # by four 10GiB systems disks; the node carries this comfortably with
            # ~810GB free on the filesystem behind it.
            sizeLimit: 150Gi
        # In memory, so the material never touches a node's disk.
        - name: secrets
          emptyDir:
            medium: Memory
            sizeLimit: 1Mi
        - name: dev-kvm
          hostPath:
            path: /dev/kvm
            type: CharDevice