Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
abb4096c67 | ||
|
|
12e1a6e6f0 | ||
|
|
b3dc3f6b4c | ||
|
|
b4de51b412 | ||
|
|
2bb2704b88 | ||
|
|
ee5f3acbaa | ||
|
|
80a0ee5b19 | ||
|
|
17bf12cc4f | ||
|
|
1fc8073218 | ||
|
|
161c04daff | ||
|
|
999e3fbac9 | ||
|
|
70208c42da | ||
|
|
94b35c3f64 | ||
|
|
5cbdb6e67f | ||
|
|
ee08edba3e | ||
|
|
96d15f19c8 | ||
|
|
0581fd461f | ||
|
|
f1024fa4b3 | ||
|
|
904475b89a | ||
|
|
c395a9fc67 | ||
|
|
bf3d58628e | ||
|
|
6edde49b48 |
@@ -84,7 +84,7 @@ steps:
|
||||
cat ../ci/agents/wiki-maintenance/distill.md | claude $${MODEL:+--model $$MODEL} \
|
||||
--agent wiki-distill \
|
||||
--permission-mode acceptEdits \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(mv /root/Notes/raw/:*),Bash(mkdir -p /root/Notes/:*)" \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(mv /root/Notes/raw/:*),Bash(mkdir -p /root/Notes/:*),Bash(~/Notes/bin/wiki_rank.py:*),Bash(~/Notes/bin/wiki_ask.py:*),Bash(mv ~/Notes/raw/:*),Bash(mkdir -p ~/Notes/:*)" \
|
||||
--settings ../ci/agents/wiki-maintenance/settings.json \
|
||||
-p
|
||||
|
||||
@@ -122,11 +122,14 @@ steps:
|
||||
cd notes
|
||||
# JSON output: the agent's report and the run's cost come out of one
|
||||
# record at the end instead of a 45-minute silent log.
|
||||
# One source for the Bash allowlist: the settings file (run 10608: the
|
||||
# agent writes ~/Notes, the rules carried only /root/Notes).
|
||||
BASH_RULES=$$(jq -r '[.permissions.allow[] | select(startswith("Bash("))] | join(",")' ../ci/agents/wiki-maintenance/consolidate-settings.json)
|
||||
rc=0
|
||||
cat ../ci/agents/wiki-maintenance/consolidate.md | claude \
|
||||
--agent wiki-consolidate \
|
||||
--permission-mode acceptEdits \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Agent,Bash(/root/Notes/bin/wiki-snapshot.sh:*),Bash(/root/Notes/bin/wiki_rank.py:*),Bash(/root/Notes/bin/wiki_ask.py:*),Bash(kubectl get:*),Bash(kubectl describe:*),Bash(kubectl version:*),Bash(kubectl api-resources:*),Bash(kubectl explain:*),Bash(git log:*),Bash(git show:*),Bash(git status:*),Bash(git diff:*),Bash(git rev-parse:*),Bash(git ls-files:*),Bash(git -C /root/source/homelab log:*),Bash(git -C /root/source/homelab show:*),Bash(git -C /root/source/homelab diff:*),Bash(git -C /root/source/homelab status:*),Bash(git -C /root/source/homelab rev-parse:*),Bash(git -C /root/source/homelab ls-files:*),Bash(git -C /root/source/homeassistant log:*),Bash(git -C /root/source/homeassistant show:*),Bash(git -C /root/source/homeassistant diff:*),Bash(git -C /root/source/homeassistant status:*),Bash(git -C /root/source/homeassistant rev-parse:*),Bash(git -C /root/source/homeassistant ls-files:*),Bash(git -C ~/source/homelab log:*),Bash(git -C ~/source/homelab show:*),Bash(git -C ~/source/homelab diff:*),Bash(git -C ~/source/homelab status:*),Bash(git -C ~/source/homelab rev-parse:*),Bash(git -C ~/source/homelab ls-files:*),Bash(git -C ~/source/homeassistant log:*),Bash(git -C ~/source/homeassistant show:*),Bash(git -C ~/source/homeassistant diff:*),Bash(git -C ~/source/homeassistant status:*),Bash(git -C ~/source/homeassistant rev-parse:*),Bash(git -C ~/source/homeassistant ls-files:*),Bash(/root/Notes/bin/wiki-delete.sh:*),Bash(ls:*),Bash(date:*),Bash(wc:*),Bash(stat:*),Bash(head:*),Bash(tail:*)" \
|
||||
--allowedTools "Read,Edit,Write,Glob,Grep,Agent,$$BASH_RULES" \
|
||||
--settings ../ci/agents/wiki-maintenance/consolidate-settings.json \
|
||||
--output-format json \
|
||||
-p > /tmp/consolidate.json || rc=$$?
|
||||
|
||||
@@ -13,6 +13,22 @@
|
||||
"Bash(/root/Notes/bin/wiki-snapshot.sh:*)",
|
||||
"Bash(/root/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(/root/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(~/Notes/bin/wiki-snapshot.sh:*)",
|
||||
"Bash(~/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(~/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(~/Notes/bin/wiki-delete.sh:*)",
|
||||
"Bash(git -C /root/Notes log:*)",
|
||||
"Bash(git -C /root/Notes show:*)",
|
||||
"Bash(git -C /root/Notes diff:*)",
|
||||
"Bash(git -C /root/Notes status:*)",
|
||||
"Bash(git -C /root/Notes rev-parse:*)",
|
||||
"Bash(git -C /root/Notes ls-files:*)",
|
||||
"Bash(git -C ~/Notes log:*)",
|
||||
"Bash(git -C ~/Notes show:*)",
|
||||
"Bash(git -C ~/Notes diff:*)",
|
||||
"Bash(git -C ~/Notes status:*)",
|
||||
"Bash(git -C ~/Notes rev-parse:*)",
|
||||
"Bash(git -C ~/Notes ls-files:*)",
|
||||
"Bash(kubectl get:*)",
|
||||
"Bash(kubectl describe:*)",
|
||||
"Bash(kubectl version:*)",
|
||||
@@ -86,6 +102,8 @@
|
||||
"Bash(git clean:*)",
|
||||
"Bash(git rebase:*)",
|
||||
"Bash(git config:*)",
|
||||
"Bash(git -C /root/Notes config:*)",
|
||||
"Bash(git -C ~/Notes config:*)",
|
||||
"Bash(git -C /root/source/homelab config:*)",
|
||||
"Bash(git -C /root/source/homeassistant config:*)",
|
||||
"Bash(git -C ~/source/homelab config:*)",
|
||||
|
||||
@@ -15,6 +15,10 @@
|
||||
"Bash(/root/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(mv /root/Notes/raw/:*)",
|
||||
"Bash(mkdir -p /root/Notes/:*)",
|
||||
"Bash(~/Notes/bin/wiki_rank.py:*)",
|
||||
"Bash(~/Notes/bin/wiki_ask.py:*)",
|
||||
"Bash(mv ~/Notes/raw/:*)",
|
||||
"Bash(mkdir -p ~/Notes/:*)",
|
||||
"Bash(ls:*)",
|
||||
"Bash(date:*)",
|
||||
"Bash(wc:*)",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
|
||||
|
||||
ARG TALOS_VERSION=v1.13.9
|
||||
ARG TALOS_VERSION=v1.13.10
|
||||
ARG VELERO_VERSION=v1.16.0
|
||||
ARG KUBECTL_VERSION=v1.36.3
|
||||
ARG FLUX_VERSION=2.9.5
|
||||
@@ -18,7 +18,7 @@ RUN apk add --no-cache \
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& echo "28e9ccd9aebd690fa03d8cf2582221d87ea38b2a1704f411d75a03589c58f2aa /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
|
||||
@@ -14,7 +14,7 @@ FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d650
|
||||
# kubernetes/talos/patches/cluster.yaml. The phase reads the version from that
|
||||
# file at run time and passes it to the provisioner; this pin only decides which
|
||||
# client talks to it.
|
||||
ARG TALOS_VERSION=v1.13.9
|
||||
ARG TALOS_VERSION=v1.13.10
|
||||
ARG KUBECTL_VERSION=v1.36.2
|
||||
ARG FLUX_VERSION=2.9.2
|
||||
ARG KOPIA_VERSION=0.23.1
|
||||
@@ -49,7 +49,7 @@ RUN apk add --no-cache \
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/talosctl \
|
||||
"https://github.com/siderolabs/talos/releases/download/${TALOS_VERSION}/talosctl-linux-amd64" \
|
||||
&& echo "7e1d4b7d5846964bdcf63a794e3c8161bb6ef2983d5ace58ea5322f3bf32a27e /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& echo "28e9ccd9aebd690fa03d8cf2582221d87ea38b2a1704f411d75a03589c58f2aa /usr/local/bin/talosctl" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/talosctl
|
||||
|
||||
RUN curl -sfL -o /usr/local/bin/kubectl \
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
# Wiki: local web interface and agent API, as a modular .NET solution
|
||||
|
||||
## Goal
|
||||
|
||||
A web interface for the Notes wiki that runs entirely in the cluster, and an
|
||||
HTTP API through which agents ask the wiki typed questions and deposit facts.
|
||||
Built as one C# solution with a MudBlazor front end, and built so that every
|
||||
external dependency is replaceable: where the notes live, which model embeds
|
||||
them, what judges a typed question, what runs an agent, how the vector index
|
||||
is stored. The first adapters are what the homelab uses today. The point of
|
||||
the design is that the next one is a project and a config line, not a
|
||||
rewrite.
|
||||
|
||||
## Where we are
|
||||
|
||||
- `wiki-index` (Python, ns `ai`) syncs `wiki/` from `k.renner/notes` by blob
|
||||
SHA, embeds through `wiki-embed` (bge-m3 on llama-server), answers
|
||||
`POST /rank`. It keeps slug, title and vector, not the text. LAN-only behind
|
||||
Traefik BasicAuth; the Gitea webhook hits the ClusterIP.
|
||||
- `~/Notes/bin/wiki_rank.py` and `wiki_ask.py` rank through the index, build a
|
||||
document context from a local vault checkout, send it with typed questions
|
||||
to TypeSafe, gate on `covered`. The wiki-synth agent composes answers from
|
||||
those judgements.
|
||||
- New knowledge enters as a file in `raw/` (facts with provenance, never
|
||||
hypotheses), distilled nightly in CI.
|
||||
- The house C# project, shelly-manager, sets the conventions: net10.0, an
|
||||
`.slnx` with `src/` and `test/`, xUnit, MudBlazor 9, Blazor Web App with
|
||||
interactive server rendering, minimal APIs, an Authentik proxy-header
|
||||
authentication handler for humans and an API-key handler for machines, a
|
||||
multi-stage Dockerfile, `dotnet test` on every push and a kaniko release
|
||||
pipeline to the Gitea registry. Human-facing apps sit behind an Authentik
|
||||
outpost (`<app>-gate`) with a CNP that admits only that outpost.
|
||||
|
||||
## Architecture: ports in the core, adapters around it
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────┐
|
||||
│ Wiki.Host (ASP.NET Core + Blazor/Mud) │ composition root,
|
||||
│ /api/v1, /rank, /webhook, UI pages │ config, auth
|
||||
└───────────────┬──────────────────────────┘
|
||||
│ uses
|
||||
┌───────────────▼──────────────────────────┐
|
||||
│ Wiki.Application (use cases) │ Sync, Rank, Ask,
|
||||
│ │ DepositFact, Queue-
|
||||
└───────────────┬──────────────────────────┘ Question, Evaluate
|
||||
│ depends only on
|
||||
┌───────────────▼──────────────────────────┐
|
||||
│ Wiki.Core (domain + ports) │ Note, RawSource,
|
||||
│ no package references │ Report, LinkGraph,
|
||||
└──────────────────────────────────────────┘ the interfaces
|
||||
implemented by ▲ ▲ ▲ ▲ ▲
|
||||
Adapters.Gitea Adapters.LlamaCpp Adapters.TypeSafe Adapters.FileIndex Adapters.Anthropic
|
||||
(later: Database, Ollama/Onnx, local Jev, pgvector/Qdrant, OpenAI-compatible answer models)
|
||||
```
|
||||
|
||||
Rules that keep it modular, enforced by project references and a test:
|
||||
|
||||
- `Wiki.Core` references nothing. It holds the domain types and the port
|
||||
interfaces. Adapters reference Core only; they never reference each other
|
||||
or the host. `Wiki.Host` is the only project that knows which adapters
|
||||
exist.
|
||||
- Every port has a **contract test suite** in `test/Wiki.Contracts` that
|
||||
runs against every adapter of that port. A new adapter is done when the
|
||||
suite is green. That is what makes "swap the storage" a safe operation.
|
||||
- Adapters are selected by configuration and registered by name (keyed
|
||||
services), so several can be active at once. The UI's lab page and the
|
||||
evaluator use that to compare, the API uses the one marked active.
|
||||
- Nothing in Core or Application knows about HTTP, Gitea, TypeSafe, files or
|
||||
Kubernetes.
|
||||
|
||||
### Ports
|
||||
|
||||
| port | what it abstracts | first adapter | examples of the next ones |
|
||||
|---|---|---|---|
|
||||
| `INoteStore` | read notes, raw sources and reports; enumerate by kind; observe changes (content hash per document) | Gitea repository API (`k.renner/notes`, blob SHA) | git checkout on a volume; a database (CNPG Postgres) with the vault imported; an Obsidian sync folder |
|
||||
| `IFactSink` | append a new raw source with provenance | Gitea contents API, new file on `main` | database insert; git commit on a checkout |
|
||||
| `IEmbeddingModel` | text → vector, with a model id and dimension | llama-server `/v1/embeddings` (bge-m3 via wiki-embed, the model that won the German-document comparison on accuracy per cost) | the qwen3 embedding variants from that comparison; ONNX in-process; any OpenAI-compatible endpoint |
|
||||
| `IVectorIndex` | store vectors per (model id, document), query top-k | in-memory cosine with a file snapshot per model, as today | pgvector in the database; Qdrant |
|
||||
| `IChunker` | split a document into addressable units for typed questions | paragraph splitter (headings, fences), as in `wiki_ask.py` | sentence windows; semantic chunks |
|
||||
| `IJudge` | typed questions over a state: Choice, Noul, Score, with probabilities | TypeSafe System One (`jev-latest`, cloud; the wiki records no locally hosted Jev today) | a locally hosted Jev with the same request shape, should TypeSafe offer one; a constrained-output local LLM adapter, flagged as uncalibrated |
|
||||
| `IAnswerModel` | prose from a question plus evidence: the selected paragraphs with their notes and the judgements; must cite by slug; streaming | Anthropic Messages API (Claude, model from config, Sonnet by default) | any OpenAI-compatible endpoint, including the local llama-server; a different Claude model per question |
|
||||
| `IEvaluator` | measure a retrieval or judge configuration against the labelled set (questions and cited sources in `output/`) | recall@k, hit rate, MRR, as `wiki_rank_eval.py` | judge accuracy against hand-labelled paragraphs |
|
||||
| `IIdentity` | who is calling, for attribution of facts and saved answers | Authentik proxy headers (humans), API key (machines), as in shelly-manager | OIDC bearer tokens |
|
||||
|
||||
Two ports deserve a sentence of scope each. The judge returns judgements
|
||||
with probabilities, never prose, and the UI never presents its output as an
|
||||
answer, only as ranked, quoted evidence. The answer model turns that
|
||||
evidence into words and is allowed nothing else: it sees the selected
|
||||
paragraphs and the judgements, not the whole context, and its output is
|
||||
shown next to the evidence it was given. Both are single model calls inside
|
||||
the request. The service runs no agent: no tool loop, no vault edits. The
|
||||
wiki-synth agent in a Claude session remains the path for artifacts and for
|
||||
multi-question synthesis, and it can call `/questions` itself.
|
||||
|
||||
### Solution layout
|
||||
|
||||
```
|
||||
Wiki.slnx
|
||||
src/
|
||||
Wiki.Core/ domain types, ports, frontmatter and link parsing
|
||||
Wiki.Application/ use cases; depends on Core only
|
||||
Wiki.Adapters.Gitea/ INoteStore, IFactSink
|
||||
Wiki.Adapters.LlamaCpp/ IEmbeddingModel (OpenAI-compatible embeddings)
|
||||
Wiki.Adapters.TypeSafe/ IJudge (System One client)
|
||||
Wiki.Adapters.FileIndex/ IVectorIndex (memory + snapshot per model)
|
||||
Wiki.Adapters.Anthropic/ IAnswerModel (Messages API, streaming)
|
||||
Wiki.Host/ ASP.NET Core: API, webhook, MudBlazor UI, auth,
|
||||
composition root, appsettings
|
||||
test/
|
||||
Wiki.Core.Tests/ parsing, link graph, chunker
|
||||
Wiki.Contracts/ one suite per port, parameterised over adapters
|
||||
Wiki.Application.Tests/ use cases with fake ports
|
||||
Wiki.Host.Tests/ API surface, auth handlers
|
||||
```
|
||||
|
||||
Own repository `k.renner/wiki`, same shape as shelly-manager: `Dockerfile`,
|
||||
`.woodpecker/test.yml` on pull requests, `.woodpecker/release.yml` building
|
||||
the image with kaniko into the Gitea registry on push to `main`, tagged
|
||||
`build.<unix-ts>` plus the commit SHA so Flux image automation can follow
|
||||
it. The homelab repo holds only the manifests. `claude-bot` needs write
|
||||
access to the new repository.
|
||||
|
||||
Two things the shelly-manager history settles in advance: the TypeSafe
|
||||
client is written from the documented field contract (endpoint, bearer key,
|
||||
`jev-latest`, Noul/Choice/Score, 255-option cap, 32k-token state budget),
|
||||
there is no prior C# client to copy; and the UI needs no login page, because
|
||||
identity comes from the outpost's headers, which avoids the Blazor-circuit
|
||||
sign-in constraint shelly-manager ran into.
|
||||
|
||||
### Configuration
|
||||
|
||||
Adapters are declared in `appsettings` and selected by name; secrets come
|
||||
through environment variables, filled by ExternalSecret, never from the file.
|
||||
|
||||
```json
|
||||
"Wiki": {
|
||||
"NoteStore": { "Active": "gitea", "gitea": { "Repository": "k.renner/notes", "Branch": "main" } },
|
||||
"Embeddings": {
|
||||
"Active": "bge-m3",
|
||||
"Models": {
|
||||
"bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc:8080", "Model": "bge-m3" },
|
||||
"e5-large": { "Adapter": "Ollama", "Url": "http://ollama.ai.svc:11434", "Model": "multilingual-e5-large" }
|
||||
}
|
||||
},
|
||||
"Judges": {
|
||||
"Active": "typesafe",
|
||||
"Models": {
|
||||
"typesafe": { "Adapter": "TypeSafe", "Url": "https://api.typesafe.ai/v1/systemone", "Model": "jev-latest" },
|
||||
"jev-local": { "Adapter": "TypeSafe", "Url": "http://jev.ai.svc:8080/v1/systemone", "Model": "jev" } // illustration: same adapter, other endpoint
|
||||
}
|
||||
},
|
||||
"Index": { "Active": "file", "file": { "Path": "/data/index" } },
|
||||
"AnswerModels": {
|
||||
"Active": "claude-sonnet",
|
||||
"Models": {
|
||||
"claude-sonnet": { "Adapter": "Anthropic", "Model": "claude-sonnet-5" },
|
||||
"qwen-local": { "Adapter": "OpenAI", "Url": "http://llama-server.ai.svc:8080/v1", "Model": "qwen3.6-35b-a3b" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Every embedding model gets its own index namespace. Switching the active
|
||||
model is a config change after the new index has been built in the
|
||||
background; until then both exist and can be compared. Nothing is
|
||||
re-embedded in place.
|
||||
|
||||
## The web interface (MudBlazor)
|
||||
|
||||
Blazor Web App, interactive server rendering, hosted in `Wiki.Host`. Static
|
||||
assets vendored into the image, no CDN. Pages:
|
||||
|
||||
1. **Search**: a query box, results ranked by the active index with score and
|
||||
tags; for a phrased question, optionally the paragraph-level hits from the
|
||||
judge with their probabilities, shown as quoted evidence.
|
||||
2. **Note**: frontmatter, rendered body with `[[wikilinks]]` resolved,
|
||||
backlinks, sources linking to their raw file.
|
||||
3. **Browse**: by tag, by `updated`, recently changed, orphans.
|
||||
4. **Answers**: a question box. The answer streams in within seconds: rank,
|
||||
judge, compose, all in the request. Below it the evidence: the paragraphs
|
||||
the judge selected with their notes and probabilities, and the `covered`
|
||||
gate; a low gate is shown as "the wiki does not cover this" instead of an
|
||||
answer. A save button writes the answer to `output/` with the wiki-synth
|
||||
frontmatter (`question`, `sources`, `confidence`), so it joins the
|
||||
labelled set. Underneath, the saved answers, newest first, including the
|
||||
ones the wiki-synth agent wrote from a session.
|
||||
5. **Maintenance**: the raw queue with ages, the last consolidation and audit
|
||||
reports, index and sync status.
|
||||
6. **Lab**: pick any registered embedding model and judge, run the same query
|
||||
through them side by side, and run the evaluator on the labelled set to
|
||||
get recall@k per configuration. This is the page for "play around with
|
||||
different models".
|
||||
7. **Settings**: read-only view of the active adapters and their health.
|
||||
|
||||
No editing of notes in the browser. The vault stays a git repository edited
|
||||
by the consolidate agent and by hand.
|
||||
|
||||
## API v1
|
||||
|
||||
Reads are authenticated by the hostname's gate (humans) or API key
|
||||
(machines); writes always need an API key so a fact carries its author.
|
||||
|
||||
| method and path | purpose |
|
||||
|---|---|
|
||||
| `GET /api/v1/status` | index commit and age, counts, active adapters |
|
||||
| `GET /api/v1/notes?q=&tag=&limit=` | list; `q` ranks through the active index |
|
||||
| `GET /api/v1/notes/{slug}` | frontmatter, body, links, backlinks, sources |
|
||||
| `GET /api/v1/tags` | tags with counts |
|
||||
| `POST /api/v1/rank` | `{question, k, model?}` → ranked slugs; also served at `/rank` for the existing scripts |
|
||||
| `POST /api/v1/ask` | `{queries, questions, k, judge?, model?}` → context, typed judgements with the answering paragraphs verbatim, `covered`. For agents and code. |
|
||||
| `POST /api/v1/questions` | `{question, k?, judge?, model?, answerModel?, save?}` → the answer text with `[[slug]]` citations, the evidence (paragraphs, probabilities), `covered`, model and token usage; streamed as server-sent events when requested; with `save`, the `output/` path and commit |
|
||||
| `POST /api/v1/facts` | deposit a raw source: title, source, `kind: fact`, observed_at, content, tags → path and commit |
|
||||
| `GET /api/v1/raw` | the distill queue with ages |
|
||||
| `GET /api/v1/reports` | `output/` documents by type, newest first |
|
||||
| `POST /api/v1/lab/eval` | run the evaluator for a (model, judge) pair |
|
||||
| `POST /webhook` | Gitea push, HMAC as today |
|
||||
|
||||
`POST /api/v1/facts` is rejected without `source`, with a `kind` other than
|
||||
`fact`, or with content under a minimum length. The raw rule is enforced by
|
||||
shape: there is no field for a hypothesis. The file gets the vault's
|
||||
provenance frontmatter with `captured_by` set from the API key's identity.
|
||||
|
||||
`POST /api/v1/ask` is `wiki_ask.py` moved server-side, so the scripts become
|
||||
thin clients and agents no longer need a vault checkout. Its result is
|
||||
judgements and quotations, never prose.
|
||||
|
||||
`POST /api/v1/questions` is `/ask` plus one answer-model call, in process:
|
||||
rank the question, build the context, run the default typed questions
|
||||
(answered, where, covered), hand the selected paragraphs and judgements to
|
||||
the answer model, stream the prose back. Below the `covered` threshold it
|
||||
returns the gap instead of an answer. Time is the sum of the parts, about
|
||||
0.2 s rank, 1 to 4 s judge, a few seconds of generation. Cost is one model
|
||||
call over a few thousand tokens, a few cents on Sonnet, against a quarter
|
||||
dollar for a full agent run.
|
||||
|
||||
## Auth and network
|
||||
|
||||
- Humans: `wiki.renner.uno`, LAN-only (no `dns.renner.uno/public` label),
|
||||
Ingress backend `ak-outpost-wiki-gate:9000`, a `wiki-gate-egress` CNP in
|
||||
`authentik`, the proxy-header handler trusts `X-Authentik-*` only from that
|
||||
outpost.
|
||||
- Machines: `wiki-index.renner.uno` with the API-key handler; the workstation
|
||||
key stays where it is. CI reaches the ClusterIP.
|
||||
- Secrets (Gitea token, webhook secret, API keys, TypeSafe key, Anthropic
|
||||
key) reach the
|
||||
pod as environment variables from Kubernetes Secrets filled by
|
||||
ExternalSecret from the `openbao` ClusterSecretStore. The pod itself has
|
||||
no OpenBao access. The Python service fetched its secrets from OpenBao
|
||||
directly through its own Kubernetes-auth role; that role, its policy and
|
||||
the `ai` → `openbao` CNP entry are retired with it.
|
||||
- Egress from the pod: Gitea, wiki-embed, `api.typesafe.ai:443`,
|
||||
`api.anthropic.com:443`. Every
|
||||
further adapter that talks to a new endpoint gets its own CNP line, which
|
||||
is the one place adding an adapter touches the cluster.
|
||||
|
||||
## Health endpoints
|
||||
|
||||
Three endpoints with three different meanings, wired to the .NET health
|
||||
checks with tags so that the probes and the humans see the right thing:
|
||||
|
||||
| endpoint | meaning | probe |
|
||||
|---|---|---|
|
||||
| `GET /health` | the process serves HTTP. Always 200 once Kestrel listens; no dependency checks. | liveness |
|
||||
| `GET /ready` | the active note store has synced at least once and the active index is loaded; 503 while the first sync or index build runs. A failed later sync keeps the last good state and stays ready. | readiness, plus a startup probe with enough budget for the first sync (the Python service was killed mid-build when its `/health` returned 503 during the initial sync) |
|
||||
| `GET /api/v1/status` | the full picture for humans and the Settings page: index commit and age, counts, queue lengths, and one entry per registered adapter with its last check result and latency. | none |
|
||||
|
||||
Adapter checks (Gitea reachable, embeddings endpoint answering, judge
|
||||
endpoint answering, answer model answering) are reported in `/status` and
|
||||
on the Settings page and are never part of `/ready`: a judge outage must not
|
||||
take the reader down. Their failures show up as red entries, not as
|
||||
restarts.
|
||||
|
||||
## Phases
|
||||
|
||||
One PR each, verified before the next. The Python `wiki-index` keeps running
|
||||
until phase 3 proves parity.
|
||||
|
||||
0. **Skeleton.** Repository, solution, `Wiki.Core` with the ports and domain
|
||||
types, `Wiki.Contracts` with the suites, `Wiki.Host` serving `/health`,
|
||||
`/ready` and `/api/v1/status`. CI: test and release pipelines. Verify:
|
||||
image in the registry, pipeline green, the Deployment's three probes
|
||||
wired and the pod Ready.
|
||||
1. **Parity with the Python service.** Gitea, LlamaCpp and FileIndex
|
||||
adapters, the sync use case, `/rank` and `/webhook`. Verify: the 40-query
|
||||
recall eval gives the same numbers as the Python service on the same
|
||||
commit; `wiki_rank.py --probe-index` against the new service passes.
|
||||
2. **Read API and UI.** Notes, tags, raw, reports; Search, Note, Browse,
|
||||
Maintenance pages; `wiki-gate`. Verify: a note renders with resolved
|
||||
links; an unauthenticated request is redirected to Authentik.
|
||||
3. **Ask.** Chunker, TypeSafe adapter, `/api/v1/ask`, scripts as clients.
|
||||
Verify: `test_wiki_ask.py` passes against the API; a wiki-synth run with
|
||||
the definition unchanged. Then the Python service is retired and the
|
||||
`wiki-index` hostname points at the new one.
|
||||
4. **Facts.** API keys as an ExternalSecret, `IFactSink`, `/api/v1/facts`. Verify:
|
||||
a fact posted from the workstation shows as a `claude-bot` commit and in
|
||||
the next distill.
|
||||
5. **Questions.** Anthropic adapter, `/api/v1/questions` with streaming,
|
||||
the Answers page with save. Verify: a question answered in under ten
|
||||
seconds with citations that resolve to the shown evidence; a question the
|
||||
wiki does not cover returns the gap; a saved answer appears in `output/`
|
||||
with the wiki-synth frontmatter and is picked up by the evaluator.
|
||||
6. **Lab.** Evaluator, a second embedding model and a second answer model
|
||||
registered, the Lab page. Verify: two models compared on the labelled set
|
||||
from the UI.
|
||||
|
||||
## Not in scope
|
||||
|
||||
- Editing notes in the browser.
|
||||
- Replacing Obsidian; the vault stays a git repository and `notes-push.path`
|
||||
keeps pushing.
|
||||
- Public exposure; LAN and VPN only.
|
||||
- Running or triggering agents. The service makes single model calls inside
|
||||
a request; it never runs a tool loop or edits the vault. Distill,
|
||||
consolidate and multi-question synthesis stay Claude processes in CI or in
|
||||
a session; the service shows their inputs and outputs and serves their API
|
||||
calls.
|
||||
@@ -107,7 +107,7 @@ config, and the nodes are addressed by name.
|
||||
## Step 1b — boot all three machines from install media
|
||||
|
||||
```
|
||||
https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.9/metal-amd64.iso
|
||||
https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.10/metal-amd64.iso
|
||||
```
|
||||
|
||||
Write it to a USB stick and boot each MS-01 from it. Each comes up in
|
||||
|
||||
@@ -6,7 +6,6 @@ kind: Kustomization
|
||||
# and verified. See MIGRATION-gitea-woodpecker.md.
|
||||
resources:
|
||||
- statefulset.yaml
|
||||
- pdb.yaml
|
||||
- service.yaml
|
||||
- ingressroute-ssh.yaml
|
||||
- ingress.yaml
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
---
|
||||
# Eviction protection for the single-replica Gitea StatefulSet.
|
||||
#
|
||||
# Gitea runs on SQLite (single writer) and its repos + DB live on a single
|
||||
# ReadWriteOnce PVC, so it is deliberately replicas:1 and cannot fail over.
|
||||
# With no PDB, a node drain / maintenance evicts the pod abruptly, mid-write,
|
||||
# with zero coordination. minAvailable:1 on a 1-replica workload makes the
|
||||
# eviction API refuse *voluntary* evictions: `kubectl drain` blocks and
|
||||
# surfaces this pod, forcing a conscious decision (drain with
|
||||
# --disable-eviction, or scale down first) instead of yanking the live DB.
|
||||
# It does NOT block involuntary disruptions (node crash, OOM).
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: gitea
|
||||
namespace: gitea
|
||||
labels:
|
||||
app.kubernetes.io/name: gitea
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: gitea
|
||||
@@ -4,7 +4,9 @@ metadata:
|
||||
name: gf-pg
|
||||
namespace: monitoring
|
||||
spec:
|
||||
instances: 1
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
|
||||
@@ -30,7 +30,9 @@ metadata:
|
||||
name: ha-pg
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
instances: 1
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
@@ -124,8 +126,7 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
# its volumes are lost outright.
|
||||
archive_timeout: "30min"
|
||||
# Fewer checkpoints => fewer full-page writes, since a page is only
|
||||
# fully-logged on its first modification after a checkpoint.
|
||||
|
||||
@@ -59,8 +59,7 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
# its volumes are lost outright.
|
||||
archive_timeout: "30min"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
|
||||
@@ -61,6 +61,9 @@ spec:
|
||||
enabled: true
|
||||
storageClass: ceph-block
|
||||
size: 50Gi
|
||||
# A minAvailable:1 PDB on a single replica blocks every node drain.
|
||||
pdb:
|
||||
create: false
|
||||
# Bootstrap admin/org/bucket on the fresh PVC. Admin password + token come
|
||||
# from the influxdb-bootstrap Secret (OpenBao via ESO), deterministic across
|
||||
# rebuilds so the bootstrap token is the live admin token. `influx setup`
|
||||
|
||||
@@ -304,6 +304,10 @@ spec:
|
||||
- matchers:
|
||||
- alertname = "KubeCPUOvercommit"
|
||||
receiver: 'null'
|
||||
# Helper alert that only drives the info inhibit rule below.
|
||||
- matchers:
|
||||
- alertname = "InfoInhibitor"
|
||||
receiver: 'null'
|
||||
# Flux notification-controller posts info-severity events
|
||||
# here (e.g. image-automation commits). Keep them visible
|
||||
# but don't repeat-spam every 12h on the same event.
|
||||
@@ -313,6 +317,23 @@ spec:
|
||||
group_wait: 1m
|
||||
group_interval: 10m
|
||||
repeat_interval: 24h
|
||||
# From kube-prometheus manifests/alertmanager-secret.yaml.
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- severity = "critical"
|
||||
target_matchers:
|
||||
- severity =~ "warning|info"
|
||||
equal: ['namespace', 'alertname']
|
||||
- source_matchers:
|
||||
- severity = "warning"
|
||||
target_matchers:
|
||||
- severity = "info"
|
||||
equal: ['namespace', 'alertname']
|
||||
- source_matchers:
|
||||
- alertname = "InfoInhibitor"
|
||||
target_matchers:
|
||||
- severity = "info"
|
||||
equal: ['namespace']
|
||||
receivers:
|
||||
- name: email
|
||||
email_configs:
|
||||
|
||||
@@ -11,7 +11,6 @@ resources:
|
||||
- protonmail-bridge.yaml
|
||||
- ftp-scanner.yaml
|
||||
- syncthing.yaml
|
||||
- pdb.yaml
|
||||
- network-policy.yaml
|
||||
# pprl-pg CNPG Postgres + data/media/consume/redis PVCs (moved here from the
|
||||
# retired seed-paperless KS; each carries prune: disabled so ownership
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
---
|
||||
# Eviction protection for the single-replica stateful paperless sidecars.
|
||||
#
|
||||
# protonmail-bridge and syncthing each hold state on a single RWO PVC and run
|
||||
# replicas:1. minAvailable:1 makes the eviction API refuse voluntary evictions,
|
||||
# so a node drain blocks and surfaces these pods instead of evicting them
|
||||
# abruptly. Involuntary disruptions (node crash) are not affected.
|
||||
#
|
||||
# NOTE: the other single-replica paperless pods (paperless, redis, gotenberg,
|
||||
# tika, ftp-scanner) are intentionally omitted — they are caches / stateless
|
||||
# converters where a blocking PDB would only wedge node drains with no
|
||||
# recoverability benefit.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: protonmail-bridge
|
||||
namespace: paperless
|
||||
labels:
|
||||
app.kubernetes.io/name: protonmail-bridge
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: protonmail-bridge
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: syncthing
|
||||
namespace: paperless
|
||||
labels:
|
||||
app.kubernetes.io/name: syncthing
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: syncthing
|
||||
@@ -4,7 +4,7 @@ metadata:
|
||||
name: tr-pg
|
||||
namespace: trmnl
|
||||
spec:
|
||||
instances: 1
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
# PG 18, NOT the repo-standard 16. Terminus ships a committed
|
||||
@@ -47,8 +47,7 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
# its volumes are lost outright.
|
||||
archive_timeout: "30min"
|
||||
# Write-reduction tuning. These databases are largely idle, and the
|
||||
# Postgres defaults favour throughput over storage that wears out.
|
||||
|
||||
@@ -22,7 +22,9 @@ metadata:
|
||||
# migration deliberately withheld this while a rollback still existed.
|
||||
kustomize.toolkit.fluxcd.io/prune: disabled
|
||||
spec:
|
||||
instances: 1
|
||||
instances: 2
|
||||
smartShutdownTimeout: 10
|
||||
stopDelay: 60
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
storage:
|
||||
storageClass: ceph-block
|
||||
@@ -91,8 +93,7 @@ spec:
|
||||
# padding rather than data.
|
||||
#
|
||||
# The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
|
||||
# its volume are lost outright. This is a single-instance cluster, so the
|
||||
# archive is the only off-node copy.
|
||||
# its volumes are lost outright.
|
||||
archive_timeout: "30min"
|
||||
resources:
|
||||
requests:
|
||||
|
||||
@@ -3,7 +3,6 @@ kind: Kustomization
|
||||
resources:
|
||||
- rbac.yaml
|
||||
- server.yaml
|
||||
- pdb.yaml
|
||||
- agent.yaml
|
||||
- service.yaml
|
||||
- ingress.yaml
|
||||
@@ -22,7 +21,7 @@ resources:
|
||||
# OpenBao access, no NetworkPolicy needed. server.yaml/agent.yaml keep their
|
||||
# Reloader annotations; externalsecret.yaml keeps the 10m refreshInterval.
|
||||
# --- namespace migration, stages 1-2 (issue #433) ---
|
||||
# Stage 2 has moved the workloads above (server, agent, service, ingress, pdb,
|
||||
# Stage 2 has moved the workloads above (server, agent, service, ingress,
|
||||
# rbac, secrets, policies) into the `woodpecker` namespace and pointed the
|
||||
# server's DSN at wp-pg-rw.woodpecker.svc. Stage 3 retired the `apps`
|
||||
# originals; these four now describe the surviving cluster. History and the
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
---
|
||||
# Eviction protection for the single-replica Woodpecker server StatefulSet.
|
||||
#
|
||||
# The server is SQLite-backed (single writer) on a single RWO PVC, so it is
|
||||
# deliberately replicas:1. minAvailable:1 makes the eviction API refuse
|
||||
# voluntary evictions so a node drain blocks and surfaces this pod rather
|
||||
# than yanking the live CI DB mid-write. Involuntary disruptions are not
|
||||
# affected.
|
||||
#
|
||||
# The selector MUST include app.kubernetes.io/component: server: the agent
|
||||
# Deployment in this same namespace shares app.kubernetes.io/name: woodpecker
|
||||
# (component: agent), and a name-only selector would wrongly also cover the
|
||||
# stateless, disposable agents.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: woodpecker-server
|
||||
namespace: woodpecker
|
||||
labels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: woodpecker
|
||||
app.kubernetes.io/component: server
|
||||
@@ -21,17 +21,29 @@ spec:
|
||||
modules:
|
||||
- name: pg_autoscaler
|
||||
enabled: true
|
||||
# Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled
|
||||
# by default. Increment keyGeneration to rotate again.
|
||||
#
|
||||
# Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and
|
||||
# these nodes run 6.18, so CSI stays on aes and the client-key checks stay
|
||||
# raised. keyType is unset so Rook picks it.
|
||||
# CephX keys on aes256k for CVE-2025-30156 (#527). Increment keyGeneration
|
||||
# to rotate again. CSI kernel mounts need aes256k in libceph: Talos
|
||||
# >= v1.13.10 backports it to 6.18 (siderolabs/pkgs 2da2cdc).
|
||||
security:
|
||||
cephx:
|
||||
# auth_allowed_ciphers. Every key must be aes256k first, or the cluster
|
||||
# locks itself out; recovery is in Rook's cephx-key-rotation doc.
|
||||
allowedCiphers:
|
||||
- aes256k
|
||||
daemon:
|
||||
keyRotationPolicy: KeyGeneration
|
||||
keyGeneration: 2
|
||||
csi:
|
||||
keyRotationPolicy: KeyGeneration
|
||||
keyGeneration: 2
|
||||
keyType: aes256k
|
||||
# 0 deletes prior keys. Set 1 before the next rotation so PVCs
|
||||
# mounted with the old key keep working until they are remounted.
|
||||
keepPriorKeyCountMax: 0
|
||||
rbdMirrorPeer:
|
||||
keyRotationPolicy: KeyGeneration
|
||||
keyGeneration: 2
|
||||
keyType: aes256k
|
||||
# CephFS cloner pool raised from the default 4. Velero clones a
|
||||
# VolumeSnapshot per PVC, and 4 threads stall a multi-PVC backup with
|
||||
# `rados: "all cloner threads are busy"`.
|
||||
|
||||
@@ -24,7 +24,7 @@ machine:
|
||||
# The Kubernetes version is a separate decision, moved by
|
||||
# `talosctl upgrade-k8s`.
|
||||
install:
|
||||
image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.13.9
|
||||
image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.13.10
|
||||
features:
|
||||
rbac: true
|
||||
# stableHostname is NOT here: it is the v1alpha1 spelling of what the
|
||||
|
||||
Reference in New Issue
Block a user