Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
00f2239a93 | ||
|
|
4fc4343329 | ||
|
|
f26d84e869 | ||
|
|
5b7d8dfaca | ||
|
|
1211d97851 | ||
|
|
682af5df88 | ||
|
|
89e011fd2e | ||
|
|
f9ca731d5d | ||
|
|
8cde6d5127 | ||
|
|
593fb6b7d9 | ||
|
|
d4a683a0ad | ||
|
|
1551523764 | ||
|
|
40362925fa | ||
|
|
0fa4b8d361 |
@@ -1,10 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Forced-command target for the netbird-backup SSH key (see the
|
||||
# "Authorize the cluster backup key" task): takes a CONSISTENT snapshot
|
||||
# of the management SQLite store via the online-backup API (safe against
|
||||
# a live writer, unlike copying store.db) and streams a gzipped tar to
|
||||
# stdout. The key's authorized_keys entry is restricted to exactly this
|
||||
# script — the in-cluster puller cannot run anything else.
|
||||
# of the management SQLite store (safe against a live writer, unlike
|
||||
# copying store.db) and streams a gzipped tar to stdout. The key's
|
||||
# authorized_keys entry is restricted to exactly this script — the
|
||||
# in-cluster puller cannot run anything else.
|
||||
#
|
||||
# Deliberately store.db ONLY: /opt/netbird/config is fully regenerable
|
||||
# from the netbird-server role's templates + vault, and including it
|
||||
@@ -14,5 +14,23 @@
|
||||
set -eu
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
|
||||
# NetBird sets no journal_mode on the store (its DSN carries only
|
||||
# _busy_timeout and cache=shared), so store.db uses the default rollback
|
||||
# journal, where the management container's write lock blocks a reader
|
||||
# outright rather than letting it through as WAL would. Two things follow.
|
||||
#
|
||||
# .timeout arms a busy handler on the source connection, which is the
|
||||
# contended side; the destination is a fresh file in $tmp that nothing else
|
||||
# touches. Without it sqlite3 uses its 0 ms default and gives up the moment
|
||||
# it cannot take the lock. 30 s sits well inside the puller's 600 s
|
||||
# activeDeadlineSeconds.
|
||||
#
|
||||
# VACUUM INTO rather than .backup: the shell's .backup copies 100 pages per
|
||||
# sqlite3_backup_step(), and SQLite restarts the copy whenever the source is
|
||||
# written between steps, so a steady writer drags it out for as long as the
|
||||
# writes keep coming. VACUUM INTO reads one transaction in a single pass and
|
||||
# is documented as an alternative to the backup API for exactly this. It
|
||||
# also compacts, so the dump is smaller than the live file.
|
||||
sqlite3 -cmd '.timeout 30000' /opt/netbird/management/store.db \
|
||||
"VACUUM INTO '$tmp/store.db'"
|
||||
tar -C "$tmp" -czf - store.db
|
||||
|
||||
@@ -119,9 +119,11 @@ against do not exist, and the node ends up with no address on any interface.
|
||||
Beyond that it carries:
|
||||
|
||||
- `siderolabs/intel-ucode` — no hypervisor supplies microcode here
|
||||
- `siderolabs/i915` — the Alder Lake-P Iris Xe. Nothing requests `/dev/dri`
|
||||
today; present so wanting hardware transcoding later is a config change
|
||||
rather than another rolling reinstall
|
||||
- `siderolabs/i915` — the Alder Lake-P Iris Xe. The Intel GPU device plugin
|
||||
(`infrastructure/intel-gpu-plugin`) turns this into the schedulable resource
|
||||
`gpu.intel.com/i915`, which `apps/llama-server` requests for LLM inference
|
||||
and which any hardware-transcoding workload would use too. Without the
|
||||
extension the plugin finds no `/dev/dri` and advertises nothing
|
||||
- `siderolabs/thunderbolt` — inert until `machine.kernel.modules` requests
|
||||
`thunderbolt`/`thunderbolt_net`, present for the same reason. Note its udev
|
||||
rule auto-authorises any Thunderbolt device attached once those modules load
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
---
|
||||
# The chat endpoint, and the only one behind the ingress.
|
||||
#
|
||||
# Single-model rather than router mode. Router mode served chat and embeddings
|
||||
# from one port, but two things made that wrong here:
|
||||
#
|
||||
# 1. Its /health answers 200 as soon as the router process listens, while the
|
||||
# models are still downloading behind it. Kubernetes marked the pod Ready
|
||||
# within seconds of start, so traffic arrived at a server that could not
|
||||
# answer. In single-model mode /health returns 503 until the model is
|
||||
# resident, which is what the probes below actually need.
|
||||
# 2. Every model on the port is selectable in the built-in WebUI, including
|
||||
# the embedding model. Asking an embedding model to chat returns fluent
|
||||
# nonsense ("capital capital capital ...") rather than an error, and
|
||||
# `--embeddings` does not prevent it. There is no flag to hide a model
|
||||
# from the list — `--tags` is informational and `--ui-config` only sets UI
|
||||
# preferences — so the only way to keep it out of the picker is to keep it
|
||||
# off this endpoint. It runs in embed.yaml instead.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llama-server
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: llama-server
|
||||
spec:
|
||||
replicas: 1
|
||||
# A rolling update would load a second copy of the weights onto the same
|
||||
# iGPU and into the same node's RAM before releasing the first.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: llama-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: llama-server
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
# Makes the CephFS volume group-writable so the weights can be cached.
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: llama-server
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
||||
args:
|
||||
# The one place this model is named; llama-server resolves the repo
|
||||
# and downloads it itself, so there is nothing to keep in sync.
|
||||
#
|
||||
# A 35B mixture-of-experts with only ~3B parameters active per
|
||||
# token. Decode on this hardware is bound by how many bytes are read
|
||||
# per token rather than by parameter count, and this model reads
|
||||
# ~1.85 GB/token where a dense 27B reads ~16.5 GB — so it is both
|
||||
# far larger and faster than any dense model that would fit.
|
||||
#
|
||||
# Measured here (llama-bench, k8s-1, this image): tg128 11.59 tok/s,
|
||||
# against 10.78 for a dense 8B. Prefill 120.9 t/s at 512 tokens and
|
||||
# 69.8 at 16384 — a sublinear falloff, so the Gated Delta Net
|
||||
# prefill collapse of llama.cpp#27638 does not occur on this stack.
|
||||
- --hf-repo
|
||||
- ggml-org/Qwen3.6-35B-A3B-GGUF:Q4_K_M
|
||||
# Every layer on the Intel iGPU. Full offload matches CPU throughput
|
||||
# while leaving the node's 14 cores free for the rest of the
|
||||
# cluster; the iGPU is here for that isolation, not for a speed-up.
|
||||
- --n-gpu-layers
|
||||
- "99"
|
||||
# Decode runs on the GPU, so extra threads buy nothing. At 11.59
|
||||
# tok/s this model moves 21.4 GB/s against the 53.9 GB/s a dense
|
||||
# model sustains here, so it is the iGPU's 96 EUs working through
|
||||
# many small per-expert matmuls that sets the pace, not bandwidth.
|
||||
- --threads
|
||||
- "4"
|
||||
# 16k rather than 8k because the measured prefill falloff is mild
|
||||
# and this model's KV cache is unusually cheap — only 10 of its 40
|
||||
# layers use full attention.
|
||||
- --ctx-size
|
||||
- "16384"
|
||||
# A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
|
||||
# with ErrorDeviceLost.
|
||||
- --ubatch-size
|
||||
- "1024"
|
||||
# The server otherwise allows every CORS origin and warns about it
|
||||
# at startup. Without this, any page a LAN browser visits could call
|
||||
# this unauthenticated API from the user's machine. Consumers talk
|
||||
# to it server-side; the WebUI is same-origin and unaffected.
|
||||
- --cors-origins
|
||||
- localhost
|
||||
# Two names for one model. Without this the id is the whole
|
||||
# --hf-repo string, which is unreadable in a model picker and in
|
||||
# logs.
|
||||
#
|
||||
# The first is what callers see: it says which model is actually
|
||||
# answering, which a bare "chat" hides and which matters as soon as
|
||||
# a second chat model exists. The second is a stable handle for
|
||||
# machine consumers — Home Assistant, Paperless, CI — so their
|
||||
# config survives the next model swap without an edit.
|
||||
- --alias
|
||||
- qwen3.6-35b-a3b,chat
|
||||
# Qwen3.6 enables thinking by default. At ~11.6 tok/s that spends
|
||||
# minutes reasoning before answering, and a request with a modest
|
||||
# max_tokens returns an empty `content` with finish_reason "length"
|
||||
# because the whole budget went to reasoning_content. Off by default
|
||||
# keeps answers usable; a client that wants it can send
|
||||
# chat_template_kwargs per request.
|
||||
- --chat-template-kwargs
|
||||
- '{"enable_thinking": false}'
|
||||
env:
|
||||
# Where --hf-repo caches the weights. Pointing it at the PVC makes
|
||||
# the download a one-time cost, shared with llama-embed, and keeps
|
||||
# the server off a home directory it cannot write as UID 1000.
|
||||
- name: LLAMA_CACHE
|
||||
value: /models
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
# Vulkan allocates the iGPU's working set from system RAM, and that
|
||||
# counts against the pod's cgroup: a 5.2 GiB model measured 7.54 GiB
|
||||
# RSS. Sized as the 19 GiB of weights plus that overhead, plus the KV
|
||||
# cache — which is unusually cheap here, since only 10 of the 40
|
||||
# layers use full attention.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 24Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
limits:
|
||||
cpu: "6"
|
||||
memory: 32Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
# /health is 503 while the model loads and 200 once it is resident, so
|
||||
# the pod only goes Ready when it can actually answer. On a cold PVC
|
||||
# that includes pulling ~19 GiB from Hugging Face, hence the long
|
||||
# startup budget (40 min); liveness takes over only after that.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
failureThreshold: 240
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llama-server-models
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: llama-server
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: llama-server
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: llama-server
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -0,0 +1,119 @@
|
||||
---
|
||||
# The embedding endpoint, for RAG and semantic search.
|
||||
#
|
||||
# Deliberately separate from llama-server and deliberately not behind the
|
||||
# ingress: an embedding model has no generative training, and when it shares a
|
||||
# port with the chat model the built-in WebUI lists it as a choice. Selecting it
|
||||
# returns fluent nonsense rather than an error, because `--embeddings` does not
|
||||
# actually refuse chat completions. Keeping it on its own ClusterIP-only Service
|
||||
# is what makes it unselectable, since llama.cpp has no way to hide a model from
|
||||
# the picker.
|
||||
#
|
||||
# Reachable in-cluster at http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: llama-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: llama-embed
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: llama-embed
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: llama-embed
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: llama-embed
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
||||
args:
|
||||
- --hf-repo
|
||||
- Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0
|
||||
# Serves POST /v1/embeddings.
|
||||
- --embeddings
|
||||
- --n-gpu-layers
|
||||
- "99"
|
||||
- --threads
|
||||
- "2"
|
||||
- --ctx-size
|
||||
- "2048"
|
||||
# No browser ever talks to this Service, so the WebUI is dead weight
|
||||
# and one more unauthenticated surface.
|
||||
- --no-webui
|
||||
env:
|
||||
# Same cache as llama-server, so the two share one copy of anything
|
||||
# they have in common and neither re-downloads the other's weights.
|
||||
- name: LLAMA_CACHE
|
||||
value: /models
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 2Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
gpu.intel.com/i915: "1"
|
||||
# ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
|
||||
# model needs, but still sized for a cold download.
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: models
|
||||
mountPath: /models
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llama-server-models
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: llama-embed
|
||||
namespace: ai
|
||||
labels:
|
||||
app.kubernetes.io/name: llama-embed
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: llama-embed
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
# LAN-only: no `dns.renner.uno/public` label, so the Cloudflare external-dns
|
||||
# instance skips it and only the UDM instance publishes the record. The API is
|
||||
# unauthenticated, so it must not be reachable from the internet.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: llama-server
|
||||
namespace: ai
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-cluster
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- llm.renner.uno
|
||||
secretName: llama-server-tls
|
||||
rules:
|
||||
- host: llm.renner.uno
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: llama-server
|
||||
port:
|
||||
number: 8080
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- embed.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
# Holds the GGUF weights, downloaded once by the Deployment's initContainer.
|
||||
# Prune-disabled so retiring this Kustomization does not throw away several
|
||||
# GiB of model files that would have to be pulled from Hugging Face again.
|
||||
#
|
||||
# ReadWriteMany so a second replica or a one-off benchmark Job can mount the
|
||||
# same weights; the models are read-only at inference time.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: llama-server-models
|
||||
namespace: ai
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: ceph-filesystem
|
||||
resources:
|
||||
requests:
|
||||
storage: 40Gi
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: llama-server
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 10m
|
||||
retryInterval: 1m
|
||||
# On a cold PVC the chat model pulls ~19 GiB of GGUF weights from Hugging
|
||||
# Face and then loads onto the iGPU before /health goes green, so a first
|
||||
# start is far slower than a normal app's.
|
||||
timeout: 60m
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: flux-system
|
||||
path: ./kubernetes/apps/llama-server
|
||||
prune: true
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: infrastructure
|
||||
- name: infrastructure-crds
|
||||
- name: infrastructure-ceph-cluster
|
||||
@@ -524,6 +524,114 @@ check "alertmanager: no severity=critical alerts active" bash -c '
|
||||
test "$body" = "[]"
|
||||
'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Local AI inference. Every check here corresponds to a defect that actually
|
||||
# shipped, because none of them are catchable by manifest validation — the
|
||||
# manifests were valid in each case and the endpoint still misbehaved.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# llm_chat: POST a chat completion through the ingress. Body lands in
|
||||
# /tmp/llm_body for the caller to assert against. Generous --max-time: the
|
||||
# model runs on an iGPU at ~12 tok/s, so a short answer still takes seconds.
|
||||
llm_chat() {
|
||||
local model="$1" prompt="$2" max="${3:-64}" out body code payload
|
||||
payload=$(jq -nc --arg m "$model" --arg p "$prompt" --argjson n "$max" \
|
||||
'{model:$m,messages:[{role:"user",content:$p}],max_tokens:$n,temperature:0}')
|
||||
out=$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 300 \
|
||||
-H 'Content-Type: application/json' -d "$payload" \
|
||||
-w $'\n__CODE__:%{http_code}' \
|
||||
"https://llm.renner.uno/v1/chat/completions" 2>&1)
|
||||
code=${out##*__CODE__:}
|
||||
body=${out%$'\n'__CODE__:*}
|
||||
printf 'code=%s body=%s\n' "$code" "${body:0:300}"
|
||||
[[ "$code" == "200" ]] || return 1
|
||||
printf '%s' "$body" > /tmp/llm_body
|
||||
}
|
||||
export -f llm_chat
|
||||
export LB_IP
|
||||
|
||||
# helper_post: POST JSON from inside e2e-helper, for Services with no ingress.
|
||||
# kubectl exec passes argv directly, so the JSON needs no shell quoting.
|
||||
helper_post() {
|
||||
local url="$1" data="$2" out body code attempt
|
||||
for attempt in 1 2 3; do
|
||||
out=$(kubectl -n monitoring exec e2e-helper -- \
|
||||
curl -sk --max-time 60 -H 'Content-Type: application/json' \
|
||||
-d "$data" -w $'\n__CODE__:%{http_code}' "$url" 2>&1)
|
||||
body=${out%$'\n'__CODE__:*}
|
||||
code=${out##*__CODE__:}
|
||||
printf 'attempt=%s code=%s body=%s\n' "$attempt" "$code" "${body:0:200}"
|
||||
if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
|
||||
printf '%s' "$body" > /tmp/helper_body
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
export -f helper_post
|
||||
|
||||
echo "=== Local AI inference ==="
|
||||
|
||||
check "llama-server /health returns 200" \
|
||||
http_code "https://llm.renner.uno/health" 200
|
||||
|
||||
# The chat model is named for what it is, so a picker and a log line say which
|
||||
# model answered. `chat` stays valid alongside it as a stable handle, so a
|
||||
# consumer's config survives the next model swap.
|
||||
check "model list advertises qwen3.6-35b-a3b" \
|
||||
http_body "https://llm.renner.uno/v1/models" 'qwen3.6-35b-a3b'
|
||||
|
||||
# Regression: the embedding model used to share this port. It appeared in the
|
||||
# chat picker, and selecting it returned fluent nonsense ("capital capital
|
||||
# capital ...") instead of an error, because --embeddings does not refuse chat
|
||||
# completions. It now lives on a ClusterIP-only Service; if it ever reappears
|
||||
# here, it is selectable again and this must fail.
|
||||
check "embedding model absent from chat endpoint" \
|
||||
bash -c '! curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
|
||||
"https://llm.renner.uno/v1/models" | grep -qi "embed"'
|
||||
|
||||
check "embeddings not exposed through the ingress" \
|
||||
http_code "https://llm.renner.uno/v1/embeddings" 501
|
||||
|
||||
# Regression: with thinking enabled by default the model spent its whole token
|
||||
# budget on reasoning_content and returned an EMPTY content with
|
||||
# finish_reason "length" — a 200 response that answers nothing. Asserting on
|
||||
# the status code alone would have passed.
|
||||
check "chat completion returns non-empty content" \
|
||||
bash -c 'llm_chat chat "Nenne die Hauptstadt von Frankreich." 64 &&
|
||||
jq -e ".choices[0].message.content | length > 0" /tmp/llm_body >/dev/null'
|
||||
|
||||
check "chat completion stops rather than truncating" \
|
||||
bash -c 'jq -e ".choices[0].finish_reason == \"stop\"" /tmp/llm_body >/dev/null'
|
||||
|
||||
# Coherence, not just liveness. This is what catches a model answering from the
|
||||
# wrong weights or a numerically broken backend — llama-bench measures
|
||||
# throughput and would report excellent numbers while emitting garbage.
|
||||
check "chat answer is coherent (knows Paris)" \
|
||||
bash -c 'jq -re ".choices[0].message.content" /tmp/llm_body | grep -qi paris'
|
||||
|
||||
# Both names must be advertised, because advertising is all that a single-model
|
||||
# server does with them: it ignores the request's `model` field entirely and
|
||||
# answers to any string, so posting a name and getting a reply proves nothing.
|
||||
# /v1/models is what a client reads to populate a picker, and it is the only
|
||||
# place the alias is observable.
|
||||
check "model list also offers the chat handle" \
|
||||
http_body "https://llm.renner.uno/v1/models" '"chat"'
|
||||
|
||||
# Reached from the monitoring namespace, which ai-ns.yaml admits; there is no
|
||||
# ingress for this Service by design.
|
||||
check "llama-embed returns an embedding vector" \
|
||||
bash -c 'helper_post "http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings" \
|
||||
"{\"model\":\"embed\",\"input\":\"Testsatz\"}" &&
|
||||
jq -e ".data[0].embedding | length > 0" /tmp/helper_body >/dev/null'
|
||||
|
||||
# Guards the resource request silently disappearing: without it the pod still
|
||||
# runs, just on CPU, and only a throughput measurement would notice.
|
||||
check "llama-server holds an iGPU allocation" \
|
||||
bash -c 'kubectl -n ai get deploy llama-server -o json |
|
||||
jq -e ".spec.template.spec.containers[0].resources.limits[\"gpu.intel.com/i915\"]" >/dev/null'
|
||||
|
||||
echo "=== Authentik OIDC discovery (per-app) ==="
|
||||
# A reachable, parseable .well-known proves Authentik has an OIDC provider
|
||||
# bound to that application slug. If the app config is missing or Authentik
|
||||
|
||||
@@ -183,3 +183,15 @@ apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: woodpecker
|
||||
---
|
||||
# Local LLM inference (kubernetes/apps/llama-server). Declared here, like
|
||||
# `woodpecker` above, so the `infrastructure` KS creates it before
|
||||
# `infrastructure-crds` applies the namespace's CiliumNetworkPolicies.
|
||||
#
|
||||
# No pod-security labels: the workload needs no hostPath or privilege — the
|
||||
# Intel GPU device plugin passes the iGPU in through the device-plugin API —
|
||||
# so the cluster's default `baseline` enforcement is enough.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: ai
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: default-deny-ingress
|
||||
namespace: ai
|
||||
spec:
|
||||
endpointSelector: {}
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ai
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ingress
|
||||
app.kubernetes.io/name: traefik
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": monitoring
|
||||
- fromEntities:
|
||||
- host
|
||||
- remote-node
|
||||
# ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
|
||||
# home-assistant-ns.yaml for the rationale (CT-RELATED path not
|
||||
# passing related ICMP empirically; per-namespace shape, mirroring
|
||||
# commit 3c517a0's verified-safe pattern with the 66dbe9f type set).
|
||||
- fromEndpoints:
|
||||
- matchExpressions:
|
||||
- key: io.kubernetes.pod.namespace
|
||||
operator: Exists
|
||||
icmps:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ai
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": ingress
|
||||
app.kubernetes.io/name: traefik
|
||||
# rook-ceph: the model weights live on a CephFS RWX volume
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": rook-ceph
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
"k8s:io.kubernetes.pod.namespace": kube-system
|
||||
k8s-app: kube-dns
|
||||
toPorts:
|
||||
- ports:
|
||||
- port: "53"
|
||||
protocol: UDP
|
||||
# host/remote-node: kubelet probes + node-local reach (Ceph mons on host net)
|
||||
# world: image pulls and llama-server's Hugging Face model download
|
||||
- toEntities:
|
||||
- host
|
||||
- remote-node
|
||||
- world
|
||||
# ICMP — egress mirror; see ingress block above.
|
||||
- toEndpoints:
|
||||
- matchExpressions:
|
||||
- key: io.kubernetes.pod.namespace
|
||||
operator: Exists
|
||||
icmps:
|
||||
- fields:
|
||||
- { type: 0, family: IPv4 }
|
||||
- { type: 3, family: IPv4 }
|
||||
- { type: 8, family: IPv4 }
|
||||
- { type: 11, family: IPv4 }
|
||||
- { type: 12, family: IPv4 }
|
||||
- { type: 1, family: IPv6 }
|
||||
- { type: 2, family: IPv6 }
|
||||
- { type: 3, family: IPv6 }
|
||||
- { type: 4, family: IPv6 }
|
||||
- { type: 128, family: IPv6 }
|
||||
- { type: 129, family: IPv6 }
|
||||
- { type: 133, family: IPv6 }
|
||||
- { type: 134, family: IPv6 }
|
||||
- { type: 135, family: IPv6 }
|
||||
- { type: 136, family: IPv6 }
|
||||
- { type: 143, family: IPv6 }
|
||||
@@ -24,4 +24,5 @@ resources:
|
||||
- woodpecker-ns.yaml
|
||||
- trmnl-ns.yaml
|
||||
- wordpress-ns.yaml
|
||||
- ai-ns.yaml
|
||||
- allow-cilium-health.yaml
|
||||
|
||||
Reference in New Issue
Block a user