diff --git a/.woodpecker/release.yml b/.woodpecker/release.yml new file mode 100644 index 0000000..2ac2e72 --- /dev/null +++ b/.woodpecker/release.yml @@ -0,0 +1,31 @@ +when: + - event: push + branch: main + +depends_on: + - test + +steps: + - name: build-image + image: gcr.io/kaniko-project/executor:v1.23.2-debug + environment: + GITEA_TOKEN: + from_secret: registry_token + commands: + - mkdir -p /kaniko/.docker + - REGISTRY=$${CI_FORGE_URL#*://} + - IMAGE="$$REGISTRY/$$CI_REPO" + # The repo owner is an organisation; the registry login is the bot in its Bots team. + - AUTH=$(printf "%s" "claude-bot:$GITEA_TOKEN" | base64 -w0) + - printf '{"auths":{"%s":{"auth":"%s"}}}' "$$REGISTRY" "$AUTH" > /kaniko/.docker/config.json + - BUILD_TAG="build.$(date +%s)" + - >- + /kaniko/executor + --context=. + --dockerfile=Dockerfile + --destination=$$IMAGE:${CI_COMMIT_SHA} + --destination=$$IMAGE:${CI_COMMIT_SHA:0:7} + --destination=$$IMAGE:$${BUILD_TAG} + --destination=$$IMAGE:main + --snapshot-mode=redo + --use-new-run diff --git a/.woodpecker/test.yml b/.woodpecker/test.yml new file mode 100644 index 0000000..8c7581d --- /dev/null +++ b/.woodpecker/test.yml @@ -0,0 +1,10 @@ +when: + - event: [pull_request, push, manual] + +steps: + - name: test + image: mcr.microsoft.com/dotnet/sdk:10.0 + commands: + - dotnet restore Wiki.slnx + - dotnet build Wiki.slnx --no-restore -c Release + - dotnet test Wiki.slnx --no-build -c Release --logger "console;verbosity=minimal" diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..9bdab74 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + true + latest + true + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..91f31b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src + +COPY Wiki.slnx Directory.Build.props ./ +COPY src/Wiki.Core/Wiki.Core.csproj src/Wiki.Core/ +COPY src/Wiki.Application/Wiki.Application.csproj src/Wiki.Application/ +COPY src/Wiki.Adapters.Gitea/Wiki.Adapters.Gitea.csproj src/Wiki.Adapters.Gitea/ +COPY src/Wiki.Adapters.LlamaCpp/Wiki.Adapters.LlamaCpp.csproj src/Wiki.Adapters.LlamaCpp/ +COPY src/Wiki.Adapters.TypeSafe/Wiki.Adapters.TypeSafe.csproj src/Wiki.Adapters.TypeSafe/ +COPY src/Wiki.Adapters.FileIndex/Wiki.Adapters.FileIndex.csproj src/Wiki.Adapters.FileIndex/ +COPY src/Wiki.Adapters.Anthropic/Wiki.Adapters.Anthropic.csproj src/Wiki.Adapters.Anthropic/ +COPY src/Wiki.Host/Wiki.Host.csproj src/Wiki.Host/ +RUN dotnet restore src/Wiki.Host/Wiki.Host.csproj + +COPY . . +# No --no-restore: publishing a Blazor Web App with it skips the static-asset +# setup and blazor.web.js never lands in the output (dotnet/aspnetcore#65291). +RUN dotnet publish src/Wiki.Host/Wiki.Host.csproj -c Release -o /app/publish /p:UseAppHost=false + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime +WORKDIR /app +COPY --from=build /app/publish ./ + +EXPOSE 8080 +ENV ASPNETCORE_URLS=http://0.0.0.0:8080 \ + DOTNET_RUNNING_IN_CONTAINER=true \ + DOTNET_NOLOGO=true + +# Index snapshots and the catalog live here; a mounted volume in the cluster. +RUN mkdir -p /data +VOLUME ["/data"] + +ENTRYPOINT ["dotnet", "Wiki.Host.dll"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..300624c --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# Wiki + +Web interface and agent API for the Notes vault (`k.renner/notes`), running in the homelab +cluster. Design: `kubernetes/PROPOSAL-wiki-web-api.md` in the homelab repository. + +## Layout + +``` +src/Wiki.Core domain types, ports, vault parsing — references nothing +src/Wiki.Application use cases (sync, rank, status, …) — depends on Core only +src/Wiki.Adapters.* one project per external thing, each references Core only +src/Wiki.Host ASP.NET Core API + MudBlazor UI, composition root +test/Wiki.Contracts one abstract suite per port; every adapter passes its suite +``` + +Adapters are selected in `appsettings.json` under `Wiki:`; secrets come from environment +variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__`). + +## Run + +``` +dotnet test Wiki.slnx +dotnet run --project src/Wiki.Host +``` + +Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`, +`POST /webhook`. diff --git a/src/Wiki.Adapters.FileIndex/FileDocumentCatalog.cs b/src/Wiki.Adapters.FileIndex/FileDocumentCatalog.cs new file mode 100644 index 0000000..f59c162 --- /dev/null +++ b/src/Wiki.Adapters.FileIndex/FileDocumentCatalog.cs @@ -0,0 +1,235 @@ +using System.Diagnostics; +using System.Text.Json; +using Microsoft.Extensions.Options; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.FileIndex; + +/// +/// An that keeps everything in memory and persists a single +/// catalog.json under the configured directory. The link graph is rebuilt from +/// on every upsert; only documents +/// contribute links and tags. +/// +public sealed class FileDocumentCatalog : IDocumentCatalog +{ + private readonly string _path; + private readonly Lock _gate = new(); + private readonly Dictionary<(DocumentKind Kind, string Slug), Document> _documents = []; + private readonly Dictionary> _outgoing = new(StringComparer.Ordinal); + private readonly Dictionary> _backlinks = new(StringComparer.Ordinal); + private CatalogState _state = new(null, null); + private bool _dirty; + private string? _resetDetail; + + public string Name => "file-catalog"; + + public FileDocumentCatalog(IOptions options) + { + Directory.CreateDirectory(options.Value.Directory); + _path = Path.Combine(options.Value.Directory, "catalog.json"); + Load(); + } + + private void Load() + { + if (!File.Exists(_path)) return; + try + { + var json = File.ReadAllText(_path); + var model = JsonSerializer.Deserialize(json); + if (model is null) return; + _state = new CatalogState(model.Commit, model.SyncedAt); + foreach (var document in model.Documents) + { + _documents[(document.Kind, document.Slug)] = document; + if (document.Kind == DocumentKind.Wiki) IndexLinks(document); + } + } + catch (Exception ex) when (ex is JsonException or IOException) + { + _resetDetail = $"catalog.json was unreadable ({ex.GetType().Name}); the catalog was reset to empty."; + _documents.Clear(); + _outgoing.Clear(); + _backlinks.Clear(); + _state = new CatalogState(null, null); + } + } + + private void IndexLinks(Document document) + { + var links = WikiLinks.Extract(document); + _outgoing[document.Slug] = [.. links]; + foreach (var link in links) + { + if (!_backlinks.TryGetValue(link.ToSlug, out var incoming)) + { + incoming = []; + _backlinks[link.ToSlug] = incoming; + } + incoming.Add(link); + } + } + + private void RemoveLinksFrom(string slug) + { + if (!_outgoing.Remove(slug, out var links)) return; + foreach (var link in links) + { + if (_backlinks.TryGetValue(link.ToSlug, out var incoming)) + { + incoming.RemoveAll(l => l.FromSlug == slug); + if (incoming.Count == 0) _backlinks.Remove(link.ToSlug); + } + } + } + + public Task GetStateAsync(CancellationToken ct) + { + lock (_gate) return Task.FromResult(_state); + } + + public Task SetStateAsync(CatalogState state, CancellationToken ct) + { + lock (_gate) + { + _state = state; + _dirty = true; + } + return Task.CompletedTask; + } + + public Task UpsertAsync(IReadOnlyList documents, CancellationToken ct) + { + lock (_gate) + { + foreach (var document in documents) + { + var key = (document.Kind, document.Slug); + if (_documents.ContainsKey(key) && document.Kind == DocumentKind.Wiki) + RemoveLinksFrom(document.Slug); + _documents[key] = document; + if (document.Kind == DocumentKind.Wiki) IndexLinks(document); + } + _dirty = true; + } + return Task.CompletedTask; + } + + public Task RemoveAsync(IReadOnlyList references, CancellationToken ct) + { + lock (_gate) + { + foreach (var reference in references) + { + _documents.Remove((reference.Kind, reference.Slug)); + if (reference.Kind == DocumentKind.Wiki) RemoveLinksFrom(reference.Slug); + } + _dirty = true; + } + return Task.CompletedTask; + } + + public Task GetAsync(DocumentKind kind, string slug, CancellationToken ct) + { + lock (_gate) return Task.FromResult(_documents.GetValueOrDefault((kind, slug))); + } + + public Task> ListAsync(DocumentKind kind, CancellationToken ct) + { + lock (_gate) + { + IReadOnlyList refs = _documents.Values.Where(d => d.Kind == kind).Select(d => d.Ref).ToList(); + return Task.FromResult(refs); + } + } + + public Task> ListDocumentsAsync(DocumentKind kind, CancellationToken ct) + { + lock (_gate) + { + IReadOnlyList docs = _documents.Values.Where(d => d.Kind == kind).ToList(); + return Task.FromResult(docs); + } + } + + public Task> BacklinksAsync(string slug, CancellationToken ct) + { + lock (_gate) + { + IReadOnlyList links = _backlinks.TryGetValue(slug, out var incoming) ? [.. incoming] : []; + return Task.FromResult(links); + } + } + + public Task> OutgoingLinksAsync(string slug, CancellationToken ct) + { + lock (_gate) + { + IReadOnlyList links = _outgoing.TryGetValue(slug, out var outgoing) ? [.. outgoing] : []; + return Task.FromResult(links); + } + } + + public Task> TagCountsAsync(CancellationToken ct) + { + lock (_gate) + { + var counts = new Dictionary(StringComparer.Ordinal); + foreach (var document in _documents.Values.Where(d => d.Kind == DocumentKind.Wiki)) + { + foreach (var tag in document.Tags) + counts[tag] = counts.GetValueOrDefault(tag) + 1; + } + return Task.FromResult>(counts); + } + } + + public Task FlushAsync(CancellationToken ct) + { + CatalogFileModel model; + lock (_gate) + { + if (!_dirty) return Task.CompletedTask; + model = new CatalogFileModel + { + Commit = _state.Commit, + SyncedAt = _state.SyncedAt, + Documents = [.. _documents.Values], + }; + _dirty = false; + } + + var tmp = _path + ".tmp"; + File.WriteAllText(tmp, JsonSerializer.Serialize(model)); + File.Move(tmp, _path, overwrite: true); + return Task.CompletedTask; + } + + public Task CheckAsync(CancellationToken ct) + { + var sw = Stopwatch.StartNew(); + try + { + var directory = Path.GetDirectoryName(_path)!; + Directory.CreateDirectory(directory); + var probe = Path.Combine(directory, $".health-{Guid.NewGuid():N}"); + File.WriteAllText(probe, string.Empty); + File.Delete(probe); + return Task.FromResult(new AdapterHealth(Name, true, _resetDetail, sw.Elapsed, DateTimeOffset.UtcNow)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return Task.FromResult(new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow)); + } + } + + private sealed class CatalogFileModel + { + public string? Commit { get; set; } + public DateTimeOffset? SyncedAt { get; set; } + public List Documents { get; set; } = []; + } +} diff --git a/src/Wiki.Adapters.FileIndex/FileIndexOptions.cs b/src/Wiki.Adapters.FileIndex/FileIndexOptions.cs new file mode 100644 index 0000000..f961f30 --- /dev/null +++ b/src/Wiki.Adapters.FileIndex/FileIndexOptions.cs @@ -0,0 +1,8 @@ +namespace Wiki.Adapters.FileIndex; + +/// Where the file-backed adapters keep their state. Shared by and . +public sealed class FileIndexOptions +{ + /// Root directory. The vector index writes under index/; the catalog writes catalog.json directly inside it. + public required string Directory { get; set; } +} diff --git a/src/Wiki.Adapters.FileIndex/FileVectorIndex.cs b/src/Wiki.Adapters.FileIndex/FileVectorIndex.cs new file mode 100644 index 0000000..564359b --- /dev/null +++ b/src/Wiki.Adapters.FileIndex/FileVectorIndex.cs @@ -0,0 +1,233 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Options; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.FileIndex; + +/// +/// An that keeps everything in memory and persists one binary file per +/// model under <Directory>/index/. Queries are frequent; upserts are batched, so a +/// guards the in-memory state rather than a single exclusive lock. +/// +public sealed class FileVectorIndex : IVectorIndex, IDisposable +{ + private const float NormTolerance = 1e-3f; + + private readonly string _indexDirectory; + private readonly ReaderWriterLockSlim _lock = new(LockRecursionPolicy.NoRecursion); + private readonly Dictionary> _models = new(StringComparer.Ordinal); + private readonly HashSet _dirtyModels = new(StringComparer.Ordinal); + private readonly List _resetNotes = []; + + public string Name => "file-index"; + + public FileVectorIndex(IOptions options) + { + _indexDirectory = Path.Combine(options.Value.Directory, "index"); + Directory.CreateDirectory(_indexDirectory); + Load(); + } + + private void Load() + { + if (!Directory.Exists(_indexDirectory)) return; + + foreach (var path in Directory.EnumerateFiles(_indexDirectory, "*.bin")) + { + try + { + using var stream = File.OpenRead(path); + using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: false); + var modelId = reader.ReadString(); + var count = reader.ReadInt32(); + var entries = new Dictionary(count, StringComparer.Ordinal); + for (var i = 0; i < count; i++) + { + var slug = reader.ReadString(); + var hash = reader.ReadString(); + var dimension = reader.ReadInt32(); + var vector = new float[dimension]; + for (var d = 0; d < dimension; d++) vector[d] = reader.ReadSingle(); + entries[slug] = new Entry(hash, vector); + } + _models[modelId] = entries; + } + catch (Exception ex) when (ex is IOException or EndOfStreamException or ArgumentOutOfRangeException or OutOfMemoryException or FormatException) + { + _resetNotes.Add($"{Path.GetFileName(path)} was unreadable ({ex.GetType().Name}); that model's index was reset to empty."); + } + } + } + + public Task DescribeAsync(string modelId, CancellationToken ct) + { + _lock.EnterReadLock(); + try + { + var hashes = _models.TryGetValue(modelId, out var entries) + ? entries.ToDictionary(kv => kv.Key, kv => kv.Value.Hash, StringComparer.Ordinal) + : new Dictionary(StringComparer.Ordinal); + return Task.FromResult(new IndexState(modelId, hashes)); + } + finally + { + _lock.ExitReadLock(); + } + } + + public Task UpsertAsync(IReadOnlyList embeddings, CancellationToken ct) + { + _lock.EnterWriteLock(); + try + { + foreach (var embedding in embeddings) + { + if (!_models.TryGetValue(embedding.ModelId, out var entries)) + { + entries = new Dictionary(StringComparer.Ordinal); + _models[embedding.ModelId] = entries; + } + entries[embedding.Slug] = new Entry(embedding.ContentHash, embedding.Vector); + _dirtyModels.Add(embedding.ModelId); + } + } + finally + { + _lock.ExitWriteLock(); + } + return Task.CompletedTask; + } + + public Task RemoveAsync(string modelId, IReadOnlyList slugs, CancellationToken ct) + { + _lock.EnterWriteLock(); + try + { + if (_models.TryGetValue(modelId, out var entries)) + { + foreach (var slug in slugs) entries.Remove(slug); + _dirtyModels.Add(modelId); + } + } + finally + { + _lock.ExitWriteLock(); + } + return Task.CompletedTask; + } + + public Task> QueryAsync(string modelId, float[] query, int k, CancellationToken ct) + { + _lock.EnterReadLock(); + try + { + if (k <= 0 || !_models.TryGetValue(modelId, out var entries) || entries.Count == 0) + return Task.FromResult>([]); + + IReadOnlyList hits = entries + .Select(kv => new RankedHit(kv.Key, Cosine(query, kv.Value.Vector))) + .OrderByDescending(h => h.Score) + .Take(k) + .ToList(); + return Task.FromResult(hits); + } + finally + { + _lock.ExitReadLock(); + } + } + + public Task FlushAsync(CancellationToken ct) + { + Dictionary> toWrite; + _lock.EnterWriteLock(); + try + { + if (_dirtyModels.Count == 0) return Task.CompletedTask; + toWrite = new Dictionary>(_dirtyModels.Count, StringComparer.Ordinal); + foreach (var modelId in _dirtyModels) + toWrite[modelId] = new Dictionary(_models.TryGetValue(modelId, out var e) ? e : [], StringComparer.Ordinal); + _dirtyModels.Clear(); + } + finally + { + _lock.ExitWriteLock(); + } + + foreach (var (modelId, entries) in toWrite) + WriteModel(modelId, entries); + return Task.CompletedTask; + } + + private void WriteModel(string modelId, Dictionary entries) + { + var path = Path.Combine(_indexDirectory, FileNameFor(modelId)); + var tmp = path + ".tmp"; + using (var stream = File.Create(tmp)) + using (var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: false)) + { + writer.Write(modelId); + writer.Write(entries.Count); + foreach (var (slug, entry) in entries) + { + writer.Write(slug); + writer.Write(entry.Hash); + writer.Write(entry.Vector.Length); + foreach (var component in entry.Vector) writer.Write(component); + } + } + File.Move(tmp, path, overwrite: true); + } + + public Task CheckAsync(CancellationToken ct) + { + var sw = Stopwatch.StartNew(); + try + { + Directory.CreateDirectory(_indexDirectory); + var probe = Path.Combine(_indexDirectory, $".health-{Guid.NewGuid():N}"); + File.WriteAllText(probe, string.Empty); + File.Delete(probe); + var detail = _resetNotes.Count == 0 ? null : string.Join(' ', _resetNotes); + return Task.FromResult(new AdapterHealth(Name, true, detail, sw.Elapsed, DateTimeOffset.UtcNow)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return Task.FromResult(new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow)); + } + } + + /// Filesystem-safe and collision-resistant: readable prefix plus a content hash of the real id. + private static string FileNameFor(string modelId) + { + var sanitized = new string(modelId.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' or '.' ? c : '_').ToArray()); + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(modelId)))[..8]; + return $"{sanitized}-{hash}.bin"; + } + + /// Cosine similarity. Vectors are unit length by contract; norms are still checked cheaply in case they are not. + private static float Cosine(float[] a, float[] b) + { + var length = Math.Min(a.Length, b.Length); + float dot = 0, normA = 0, normB = 0; + for (var i = 0; i < length; i++) + { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + normA = MathF.Sqrt(normA); + normB = MathF.Sqrt(normB); + if (MathF.Abs(normA - 1f) <= NormTolerance && MathF.Abs(normB - 1f) <= NormTolerance) + return dot; + var denom = normA * normB; + return denom > 0 ? dot / denom : 0f; + } + + public void Dispose() => _lock.Dispose(); + + private sealed record Entry(string Hash, float[] Vector); +} diff --git a/src/Wiki.Adapters.FileIndex/ServiceCollectionExtensions.cs b/src/Wiki.Adapters.FileIndex/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..ef0f53e --- /dev/null +++ b/src/Wiki.Adapters.FileIndex/ServiceCollectionExtensions.cs @@ -0,0 +1,21 @@ +using Microsoft.Extensions.DependencyInjection; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.FileIndex; + +public static class ServiceCollectionExtensions +{ + /// Registers and as singletons sharing one . + public static IServiceCollection AddFileIndexAdapters(this IServiceCollection services, Action configure) + { + services.Configure(configure); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + + return services; + } +} diff --git a/src/Wiki.Adapters.FileIndex/Wiki.Adapters.FileIndex.csproj b/src/Wiki.Adapters.FileIndex/Wiki.Adapters.FileIndex.csproj index f5a3032..56c99ae 100644 --- a/src/Wiki.Adapters.FileIndex/Wiki.Adapters.FileIndex.csproj +++ b/src/Wiki.Adapters.FileIndex/Wiki.Adapters.FileIndex.csproj @@ -4,6 +4,11 @@ + + + + + net10.0 enable diff --git a/src/Wiki.Adapters.Gitea/GiteaDtos.cs b/src/Wiki.Adapters.Gitea/GiteaDtos.cs new file mode 100644 index 0000000..417b413 --- /dev/null +++ b/src/Wiki.Adapters.Gitea/GiteaDtos.cs @@ -0,0 +1,58 @@ +using System.Text.Json.Serialization; + +namespace Wiki.Adapters.Gitea; + +// Shapes of the Gitea API responses actually used. Verified against a live instance +// (GET /repos/{o}/{r}/branches/{b}, GET /repos/{o}/{r}/git/trees/{sha}) rather than the swagger doc. + +internal sealed class BranchResponse +{ + [JsonPropertyName("commit")] + public BranchCommit Commit { get; set; } = new(); +} + +internal sealed class BranchCommit +{ + [JsonPropertyName("id")] + public string Id { get; set; } = ""; + + [JsonPropertyName("timestamp")] + public DateTimeOffset? Timestamp { get; set; } +} + +internal sealed class TreeResponse +{ + [JsonPropertyName("tree")] + public List? Tree { get; set; } + + [JsonPropertyName("total_count")] + public int TotalCount { get; set; } + + [JsonPropertyName("truncated")] + public bool Truncated { get; set; } +} + +internal sealed class TreeEntry +{ + [JsonPropertyName("path")] + public string Path { get; set; } = ""; + + [JsonPropertyName("type")] + public string Type { get; set; } = ""; + + [JsonPropertyName("sha")] + public string Sha { get; set; } = ""; +} + +/// The subset of Gitea's "create file" response (POST .../contents/{path}) that a fact sink needs. +internal sealed class ContentsCreateResponse +{ + [JsonPropertyName("commit")] + public ContentsCommit Commit { get; set; } = new(); +} + +internal sealed class ContentsCommit +{ + [JsonPropertyName("sha")] + public string Sha { get; set; } = ""; +} diff --git a/src/Wiki.Adapters.Gitea/GiteaFactSink.cs b/src/Wiki.Adapters.Gitea/GiteaFactSink.cs new file mode 100644 index 0000000..a5ff3eb --- /dev/null +++ b/src/Wiki.Adapters.Gitea/GiteaFactSink.cs @@ -0,0 +1,157 @@ +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using Microsoft.Extensions.Options; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.Gitea; + +/// Deposits a fact as a new file under raw/ in Gitea. Never edits an existing file: a +/// title that would collide with an existing path at HEAD gets a numbered suffix instead. +public sealed class GiteaFactSink : IFactSink +{ + private const int MaxSlugLength = 80; + private const int MaxSuffixAttempts = 1000; + + private readonly HttpClient _http; + private readonly GiteaOptions _options; + + public string Name => "gitea"; + + public GiteaFactSink(HttpClient httpClient, IOptions options) + { + _http = httpClient; + _options = options.Value; + GiteaHttp.Configure(_http, _options); + } + + public async Task AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct) + { + var errors = fact.Validate(); + if (errors.Count > 0) + throw new ArgumentException($"Invalid raw source: {string.Join("; ", errors)}", nameof(fact)); + + var datePrefix = now.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + var slug = Slugify(fact.Title); + var path = await ReservePathAsync(datePrefix, slug, ct).ConfigureAwait(false); + var text = BuildFile(fact, by, now, datePrefix); + + var repo = GiteaUrls.EncodeSegments(_options.Repository); + var encodedPath = GiteaUrls.EncodeSegments(path); + var request = new + { + content = Convert.ToBase64String(Encoding.UTF8.GetBytes(text)), + message = $"raw: {fact.Title}", + branch = _options.Branch, + }; + using var response = await _http.PostAsJsonAsync($"api/v1/repos/{repo}/contents/{encodedPath}", request, GiteaHttp.Json, ct) + .ConfigureAwait(false); + await GiteaHttp.EnsureSuccessAsync(response, ct).ConfigureAwait(false); + var created = await response.Content.ReadFromJsonAsync(GiteaHttp.Json, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException("Gitea returned an empty body creating a file."); + return new FactReceipt(path, created.Commit.Sha, now); + } + + public async Task CheckAsync(CancellationToken ct) + { + var sw = Stopwatch.StartNew(); + try + { + var repo = GiteaUrls.EncodeSegments(_options.Repository); + var branch = Uri.EscapeDataString(_options.Branch); + var head = await GiteaHttp.GetJsonAsync(_http, $"api/v1/repos/{repo}/branches/{branch}", ct) + .ConfigureAwait(false); + return new AdapterHealth(Name, true, $"HEAD {head.Commit.Id}", sw.Elapsed, DateTimeOffset.UtcNow); + } + catch (Exception ex) + { + return new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow); + } + } + + /// The candidate path if free at HEAD, else the same with -2, -3, ... appended until one is free. + private async Task ReservePathAsync(string datePrefix, string slug, CancellationToken ct) + { + var candidate = $"raw/{datePrefix}-{slug}.md"; + for (var attempt = 2; await ExistsAtHeadAsync(candidate, ct).ConfigureAwait(false); attempt++) + { + if (attempt > MaxSuffixAttempts) + throw new InvalidOperationException($"Gitea has {MaxSuffixAttempts}+ raw files named like {datePrefix}-{slug}; giving up."); + candidate = $"raw/{datePrefix}-{slug}-{attempt}.md"; + } + return candidate; + } + + private async Task ExistsAtHeadAsync(string path, CancellationToken ct) + { + var repo = GiteaUrls.EncodeSegments(_options.Repository); + var encodedPath = GiteaUrls.EncodeSegments(path); + var branch = Uri.EscapeDataString(_options.Branch); + using var response = await _http.GetAsync($"api/v1/repos/{repo}/contents/{encodedPath}?ref={branch}", ct).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) return false; + await GiteaHttp.EnsureSuccessAsync(response, ct).ConfigureAwait(false); + return true; + } + + private static string BuildFile(RawSource fact, Identity by, DateTimeOffset now, string capturedDate) + { + var body = fact.Content; + if (!body.EndsWith('\n')) body += "\n"; + + var sb = new StringBuilder(); + sb.Append("---\n"); + sb.Append("title: ").Append(YamlScalar(fact.Title)).Append('\n'); + sb.Append("type: raw\n"); + sb.Append("source: ").Append(YamlScalar(fact.Source)).Append('\n'); + sb.Append("captured: ").Append(capturedDate).Append('\n'); + sb.Append("captured_by: ").Append(by.Name).Append('\n'); + sb.Append("observed: ").Append(fact.ObservedAt.ToString("yyyy-MM-ddTHH:mm:ssK", CultureInfo.InvariantCulture)).Append('\n'); + sb.Append("tags: [").Append(string.Join(", ", fact.Tags)).Append("]\n"); + sb.Append("---\n"); + sb.Append('\n'); + sb.Append(body); + return sb.ToString(); + } + + /// Quotes a scalar that would otherwise be ambiguous or wrong in flow-less YAML: a colon + /// or a '#' can start a mapping value or a comment mid-line, and a leading quote needs escaping. + private static string YamlScalar(string value) + { + if (!value.Contains(':') && !value.Contains('#') && !value.StartsWith('"') && !value.StartsWith('\'')) + return value; + return $"\"{value.Replace("\"", "\\\"")}\""; + } + + private static string Slugify(string title) + { + var sb = new StringBuilder(title.Length); + foreach (var ch in title.ToLowerInvariant()) + { + if (ch is (>= 'a' and <= 'z') or (>= '0' and <= '9')) sb.Append(ch); + else sb.Append('-'); + } + + var collapsed = new StringBuilder(sb.Length); + var lastWasDash = false; + foreach (var ch in sb.ToString()) + { + if (ch == '-') + { + if (lastWasDash) continue; + lastWasDash = true; + } + else + { + lastWasDash = false; + } + collapsed.Append(ch); + } + + var slug = collapsed.ToString().Trim('-'); + if (slug.Length > MaxSlugLength) slug = slug[..MaxSlugLength].Trim('-'); + return slug.Length == 0 ? "untitled" : slug; + } +} diff --git a/src/Wiki.Adapters.Gitea/GiteaHttp.cs b/src/Wiki.Adapters.Gitea/GiteaHttp.cs new file mode 100644 index 0000000..bf43add --- /dev/null +++ b/src/Wiki.Adapters.Gitea/GiteaHttp.cs @@ -0,0 +1,42 @@ +using System.Net.Http.Headers; +using System.Text.Json; + +namespace Wiki.Adapters.Gitea; + +/// HTTP plumbing shared by and : +/// client setup, JSON decoding and turning a failed response into a message worth reading. +internal static class GiteaHttp +{ + public static readonly JsonSerializerOptions Json = new(); + + private const int ErrorBodyPreviewLength = 300; + + /// Points the client at the configured Gitea and carries the token, unless the caller + /// (a test, typically) has already set these up on the HttpClient it handed in. + public static void Configure(HttpClient client, GiteaOptions options) + { + client.BaseAddress ??= new Uri(options.BaseUrl, UriKind.Absolute); + if (client.DefaultRequestHeaders.Authorization is null && !string.IsNullOrEmpty(options.Token)) + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("token", options.Token); + } + + public static async Task GetJsonAsync(HttpClient client, string url, CancellationToken ct) + { + using var response = await client.GetAsync(url, ct).ConfigureAwait(false); + await EnsureSuccessAsync(response, ct).ConfigureAwait(false); + await using var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + return await JsonSerializer.DeserializeAsync(stream, Json, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Gitea returned an empty body for {url}."); + } + + public static async Task EnsureSuccessAsync(HttpResponseMessage response, CancellationToken ct) + { + if (response.IsSuccessStatusCode) return; + var body = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + if (body.Length > ErrorBodyPreviewLength) body = body[..ErrorBodyPreviewLength]; + throw new HttpRequestException( + $"Gitea {(int)response.StatusCode} {response.ReasonPhrase} for {response.RequestMessage?.RequestUri}: {body}", + null, + response.StatusCode); + } +} diff --git a/src/Wiki.Adapters.Gitea/GiteaNoteStore.cs b/src/Wiki.Adapters.Gitea/GiteaNoteStore.cs new file mode 100644 index 0000000..7e83748 --- /dev/null +++ b/src/Wiki.Adapters.Gitea/GiteaNoteStore.cs @@ -0,0 +1,97 @@ +using System.Diagnostics; +using Microsoft.Extensions.Options; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.Gitea; + +/// Reads the vault straight from a Gitea repository: HEAD, a recursive tree listing and raw +/// file contents, all at the commit the listing was taken from. +public sealed class GiteaNoteStore : INoteStore +{ + // Gitea answers "tree": null past the last page, and may mark the last page truncated=true even + // though nothing follows; stop on a short/empty page or once total_count is reached either way. + private const int PageSize = 1000; + private const int MaxPages = 50; + + private readonly HttpClient _http; + private readonly GiteaOptions _options; + + public string Name => "gitea"; + + public GiteaNoteStore(HttpClient httpClient, IOptions options) + { + _http = httpClient; + _options = options.Value; + GiteaHttp.Configure(_http, _options); + } + + public async Task GetHeadAsync(CancellationToken ct) + { + var branch = await FetchBranchAsync(ct).ConfigureAwait(false); + return new StoreHead(branch.Commit.Id, branch.Commit.Timestamp); + } + + public async Task> ListAsync(string commit, IReadOnlySet kinds, CancellationToken ct) + { + var repo = GiteaUrls.EncodeSegments(_options.Repository); + var refs = new List(); + var page = 1; + var seen = 0; + while (true) + { + var url = $"api/v1/repos/{repo}/git/trees/{Uri.EscapeDataString(commit)}?recursive=true&per_page={PageSize}&page={page}"; + var tree = await GiteaHttp.GetJsonAsync(_http, url, ct).ConfigureAwait(false); + var entries = tree.Tree ?? []; + foreach (var entry in entries) + { + if (entry.Type != "blob") continue; + if (DocumentParser.Classify(entry.Path) is not { } classified) continue; + if (!kinds.Contains(classified.Kind)) continue; + refs.Add(new DocumentRef(classified.Kind, classified.Slug, entry.Path, entry.Sha)); + } + seen += entries.Count; + // total_count is the reliable signal (confirmed live: Gitea reports it regardless of the + // per_page requested); a short page only means "done" when total_count isn't available, + // since a server may cap the effective page size below what was asked for. + var done = entries.Count == 0 + || (tree.TotalCount > 0 ? seen >= tree.TotalCount : entries.Count < PageSize); + if (done) break; + page++; + if (page > MaxPages) throw new InvalidOperationException("Gitea tree paging exceeded the safety cap."); + } + return refs; + } + + public async Task ReadAsync(DocumentRef reference, string commit, CancellationToken ct) + { + var repo = GiteaUrls.EncodeSegments(_options.Repository); + var path = GiteaUrls.EncodeSegments(reference.Path); + var url = $"api/v1/repos/{repo}/raw/{path}?ref={Uri.EscapeDataString(commit)}"; + using var response = await _http.GetAsync(url, ct).ConfigureAwait(false); + await GiteaHttp.EnsureSuccessAsync(response, ct).ConfigureAwait(false); + return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + } + + public async Task CheckAsync(CancellationToken ct) + { + var sw = Stopwatch.StartNew(); + try + { + var branch = await FetchBranchAsync(ct).ConfigureAwait(false); + return new AdapterHealth(Name, true, $"HEAD {branch.Commit.Id}", sw.Elapsed, DateTimeOffset.UtcNow); + } + catch (Exception ex) + { + return new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow); + } + } + + private Task FetchBranchAsync(CancellationToken ct) + { + var repo = GiteaUrls.EncodeSegments(_options.Repository); + var branch = Uri.EscapeDataString(_options.Branch); + return GiteaHttp.GetJsonAsync(_http, $"api/v1/repos/{repo}/branches/{branch}", ct); + } +} diff --git a/src/Wiki.Adapters.Gitea/GiteaOptions.cs b/src/Wiki.Adapters.Gitea/GiteaOptions.cs new file mode 100644 index 0000000..9365cb0 --- /dev/null +++ b/src/Wiki.Adapters.Gitea/GiteaOptions.cs @@ -0,0 +1,10 @@ +namespace Wiki.Adapters.Gitea; + +/// How to reach the notes repository on Gitea. +public sealed class GiteaOptions +{ + public string BaseUrl { get; set; } = "https://gitea.renner.uno"; + public string Repository { get; set; } = "k.renner/notes"; + public string Branch { get; set; } = "main"; + public string Token { get; set; } = ""; +} diff --git a/src/Wiki.Adapters.Gitea/GiteaUrls.cs b/src/Wiki.Adapters.Gitea/GiteaUrls.cs new file mode 100644 index 0000000..1b00757 --- /dev/null +++ b/src/Wiki.Adapters.Gitea/GiteaUrls.cs @@ -0,0 +1,9 @@ +namespace Wiki.Adapters.Gitea; + +/// Path building shared by the store and the sink: repo/branch/path segments are each +/// percent-encoded on their own so a literal '/' between segments survives, matching Gitea's routing. +internal static class GiteaUrls +{ + public static string EncodeSegments(string value) => + string.Join('/', value.Split('/').Select(Uri.EscapeDataString)); +} diff --git a/src/Wiki.Adapters.Gitea/ServiceCollectionExtensions.cs b/src/Wiki.Adapters.Gitea/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..c2a8ff6 --- /dev/null +++ b/src/Wiki.Adapters.Gitea/ServiceCollectionExtensions.cs @@ -0,0 +1,24 @@ +using Microsoft.Extensions.DependencyInjection; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.Gitea; + +public static class ServiceCollectionExtensions +{ + /// Registers the Gitea-backed and , each + /// with its own managed HttpClient from . Reads retry transient + /// failures; pooled connections are recycled so a stale keep-alive is not reused for long. + public static IServiceCollection AddGiteaAdapters(this IServiceCollection services, Action configure) + { + services.Configure(configure); + services.AddHttpClient() + .AddHttpMessageHandler(() => new TransientRetryHandler()) + .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(2) }); + services.AddHttpClient() + .AddHttpMessageHandler(() => new TransientRetryHandler()) + .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(2) }); + services.AddTransient(sp => sp.GetRequiredService()); + services.AddTransient(sp => sp.GetRequiredService()); + return services; + } +} diff --git a/src/Wiki.Adapters.Gitea/TransientRetryHandler.cs b/src/Wiki.Adapters.Gitea/TransientRetryHandler.cs new file mode 100644 index 0000000..128c43e --- /dev/null +++ b/src/Wiki.Adapters.Gitea/TransientRetryHandler.cs @@ -0,0 +1,49 @@ +using System.Net; + +namespace Wiki.Adapters.Gitea; + +/// +/// Retries idempotent requests (GET, HEAD) on transport failures and 5xx responses with a short +/// backoff. A sync reads thousands of documents over one keep-alive connection; Gitea behind +/// Traefik resets such a connection now and then, and one reset must not abort the whole run. +/// Non-idempotent requests (the fact sink's POST) are never retried here. +/// +public sealed class TransientRetryHandler(int attempts = 3, TimeSpan? firstDelay = null, Func? delay = null) : DelegatingHandler +{ + private readonly TimeSpan _firstDelay = firstDelay ?? TimeSpan.FromMilliseconds(500); + private readonly Func _delay = delay ?? Task.Delay; + + public int Attempts { get; } = Math.Max(1, attempts); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + var idempotent = request.Method == HttpMethod.Get || request.Method == HttpMethod.Head; + var wait = _firstDelay; + for (var attempt = 1; ; attempt++) + { + HttpResponseMessage? response = null; + try + { + response = await base.SendAsync(request, ct).ConfigureAwait(false); + if (!idempotent || attempt >= Attempts || !IsTransient(response.StatusCode)) + return response; + } + catch (HttpRequestException) when (idempotent && attempt < Attempts) + { + } + catch (IOException) when (idempotent && attempt < Attempts) + { + } + catch (TaskCanceledException) when (idempotent && attempt < Attempts && !ct.IsCancellationRequested) + { + // HttpClient timeout, not the caller cancelling + } + response?.Dispose(); + await _delay(wait, ct).ConfigureAwait(false); + wait += wait; + } + } + + private static bool IsTransient(HttpStatusCode status) => + status is HttpStatusCode.BadGateway or HttpStatusCode.ServiceUnavailable or HttpStatusCode.GatewayTimeout or HttpStatusCode.InternalServerError; +} diff --git a/src/Wiki.Adapters.Gitea/Wiki.Adapters.Gitea.csproj b/src/Wiki.Adapters.Gitea/Wiki.Adapters.Gitea.csproj index f5a3032..9440b48 100644 --- a/src/Wiki.Adapters.Gitea/Wiki.Adapters.Gitea.csproj +++ b/src/Wiki.Adapters.Gitea/Wiki.Adapters.Gitea.csproj @@ -4,6 +4,11 @@ + + + + + net10.0 enable diff --git a/src/Wiki.Adapters.LlamaCpp/EmbeddingWireModel.cs b/src/Wiki.Adapters.LlamaCpp/EmbeddingWireModel.cs new file mode 100644 index 0000000..5ef17c1 --- /dev/null +++ b/src/Wiki.Adapters.LlamaCpp/EmbeddingWireModel.cs @@ -0,0 +1,10 @@ +namespace Wiki.Adapters.LlamaCpp; + +/// Request body for POST /v1/embeddings. +internal sealed record EmbeddingRequest(string Model, IReadOnlyList Input); + +/// Response body from POST /v1/embeddings. +internal sealed record EmbeddingResponse(IReadOnlyList Data); + +/// One row of an embeddings response; rows are not guaranteed to arrive in request order. +internal sealed record EmbeddingRow(int Index, float[] Embedding); diff --git a/src/Wiki.Adapters.LlamaCpp/LlamaCppEmbeddingModel.cs b/src/Wiki.Adapters.LlamaCpp/LlamaCppEmbeddingModel.cs new file mode 100644 index 0000000..c69bf19 --- /dev/null +++ b/src/Wiki.Adapters.LlamaCpp/LlamaCppEmbeddingModel.cs @@ -0,0 +1,126 @@ +using System.Diagnostics; +using System.Net; +using System.Text; +using System.Text.Json; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.LlamaCpp; + +/// Embeds text via a llama.cpp server's OpenAI-compatible /v1/embeddings endpoint. +public sealed class LlamaCppEmbeddingModel : IEmbeddingModel +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _http; + private readonly LlamaCppEmbeddingOptions _options; + + public LlamaCppEmbeddingModel(HttpClient http, LlamaCppEmbeddingOptions options) + { + _http = http; + _options = options; + } + + public string ModelId => _options.ModelId; + + public int Dimension => _options.Dimension; + + public string Name => $"llamacpp-embeddings:{_options.ModelId}"; + + public async Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) + { + if (texts.Count == 0) + { + return []; + } + + var result = new float[texts.Count][]; + for (var offset = 0; offset < texts.Count; offset += _options.BatchSize) + { + var batch = texts.Skip(offset).Take(_options.BatchSize).ToArray(); + var vectors = await EmbedBatchAsync(batch, ct).ConfigureAwait(false); + for (var i = 0; i < vectors.Length; i++) + { + result[offset + i] = vectors[i]; + } + } + + return result; + } + + public async Task CheckAsync(CancellationToken ct) + { + var stopwatch = Stopwatch.StartNew(); + try + { + using var response = await _http.GetAsync("health", ct).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) + { + await EmbedBatchAsync(["health"], ct).ConfigureAwait(false); + return new AdapterHealth(Name, true, "no /health endpoint; verified via /v1/embeddings", stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + + if (response.IsSuccessStatusCode) + { + return new AdapterHealth(Name, true, null, stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + + return new AdapterHealth(Name, false, $"GET /health returned {(int)response.StatusCode}", stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException) + { + return new AdapterHealth(Name, false, ex.Message, stopwatch.Elapsed, DateTimeOffset.UtcNow); + } + } + + private async Task EmbedBatchAsync(IReadOnlyList batch, CancellationToken ct) + { + var request = new EmbeddingRequest(_options.Model, batch); + var json = JsonSerializer.Serialize(request, JsonOptions); + using var content = new StringContent(json, Encoding.UTF8, "application/json"); + using var response = await _http.PostAsync("v1/embeddings", content, ct).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var body = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); + var payload = await JsonSerializer.DeserializeAsync(body, JsonOptions, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException("llama.cpp embeddings response body was empty."); + + var ordered = payload.Data.OrderBy(row => row.Index).ToArray(); + var vectors = new float[ordered.Length][]; + for (var i = 0; i < ordered.Length; i++) + { + vectors[i] = Normalize(ordered[i].Embedding); + } + + return vectors; + } + + private float[] Normalize(float[] vector) + { + if (vector.Length != _options.Dimension) + { + throw new InvalidOperationException( + $"llama.cpp returned an embedding of dimension {vector.Length}, expected {_options.Dimension}."); + } + + var sumSquares = 0.0; + foreach (var x in vector) + { + sumSquares += (double)x * x; + } + + var norm = Math.Sqrt(sumSquares); + if (norm == 0) + { + return vector; + } + + var result = new float[vector.Length]; + for (var i = 0; i < vector.Length; i++) + { + result[i] = (float)(vector[i] / norm); + } + + return result; + } +} diff --git a/src/Wiki.Adapters.LlamaCpp/LlamaCppEmbeddingOptions.cs b/src/Wiki.Adapters.LlamaCpp/LlamaCppEmbeddingOptions.cs new file mode 100644 index 0000000..82202e9 --- /dev/null +++ b/src/Wiki.Adapters.LlamaCpp/LlamaCppEmbeddingOptions.cs @@ -0,0 +1,55 @@ +namespace Wiki.Adapters.LlamaCpp; + +/// Configuration for one llama.cpp /v1/embeddings endpoint. +public sealed class LlamaCppEmbeddingOptions +{ + private string? _modelId; + + /// Base URL of the llama-server, e.g. http://wiki-embed.ai.svc.cluster.local:8080. + public string Url { get; set; } = ""; + + /// Model name sent in the request body, e.g. "bge-m3". + public string Model { get; set; } = ""; + + /// Identifier used as the index namespace. Defaults to . + public string ModelId + { + get => string.IsNullOrWhiteSpace(_modelId) ? Model : _modelId; + set => _modelId = value; + } + + /// Length every returned embedding must have. + public int Dimension { get; set; } + + /// Requests are chunked to this many texts; llama-server's embedding endpoint chokes on larger batches for long notes. + public int BatchSize { get; set; } = 2; + + /// Optional "user:password" sent as HTTP Basic auth, for reaching the service through the Traefik BasicAuth ingress from outside the cluster. + public string? BasicAuth { get; set; } + + /// Per-request timeout. + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(120); + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(Url)) + { + throw new ArgumentException("Url must be set.", nameof(Url)); + } + + if (string.IsNullOrWhiteSpace(Model)) + { + throw new ArgumentException("Model must be set.", nameof(Model)); + } + + if (Dimension <= 0) + { + throw new ArgumentException("Dimension must be positive.", nameof(Dimension)); + } + + if (BatchSize <= 0) + { + throw new ArgumentException("BatchSize must be positive.", nameof(BatchSize)); + } + } +} diff --git a/src/Wiki.Adapters.LlamaCpp/ServiceCollectionExtensions.cs b/src/Wiki.Adapters.LlamaCpp/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..48aa7dc --- /dev/null +++ b/src/Wiki.Adapters.LlamaCpp/ServiceCollectionExtensions.cs @@ -0,0 +1,50 @@ +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Wiki.Core.Ports; + +namespace Wiki.Adapters.LlamaCpp; + +public static class ServiceCollectionExtensions +{ + /// + /// Registers an backed by a llama.cpp server, keyed by . + /// Several models may be registered under different names, each with its own . + /// + public static IServiceCollection AddLlamaCppEmbeddings( + this IServiceCollection services, + string name, + Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(configure); + + var options = new LlamaCppEmbeddingOptions(); + configure(options); + options.Validate(); + + var httpClientName = $"llamacpp-embeddings:{name}"; + + services.AddHttpClient(httpClientName, client => + { + client.BaseAddress = new Uri(options.Url.TrimEnd('/') + "/"); + client.Timeout = options.Timeout; + + if (!string.IsNullOrEmpty(options.BasicAuth)) + { + var token = Convert.ToBase64String(Encoding.UTF8.GetBytes(options.BasicAuth)); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", token); + } + }); + + services.AddKeyedSingleton(name, (sp, _) => + { + var httpClientFactory = sp.GetRequiredService(); + var client = httpClientFactory.CreateClient(httpClientName); + return new LlamaCppEmbeddingModel(client, options); + }); + + return services; + } +} diff --git a/src/Wiki.Adapters.LlamaCpp/Wiki.Adapters.LlamaCpp.csproj b/src/Wiki.Adapters.LlamaCpp/Wiki.Adapters.LlamaCpp.csproj index f5a3032..9440b48 100644 --- a/src/Wiki.Adapters.LlamaCpp/Wiki.Adapters.LlamaCpp.csproj +++ b/src/Wiki.Adapters.LlamaCpp/Wiki.Adapters.LlamaCpp.csproj @@ -4,6 +4,11 @@ + + + + + net10.0 enable diff --git a/src/Wiki.Application/Retrieval/RankUseCase.cs b/src/Wiki.Application/Retrieval/RankUseCase.cs new file mode 100644 index 0000000..d7b8bba --- /dev/null +++ b/src/Wiki.Application/Retrieval/RankUseCase.cs @@ -0,0 +1,27 @@ +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Retrieval; + +public sealed record RankedNote(string Slug, string Title, float Score); + +/// Embed the question with the chosen model, query its index namespace, resolve titles. +public sealed class RankUseCase(IVectorIndex index, IDocumentCatalog catalog) +{ + public const int MaxK = 500; + + public async Task> RankAsync(IEmbeddingModel model, string question, int k, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(question)) throw new ArgumentException("question required", nameof(question)); + k = Math.Clamp(k, 1, MaxK); + var vector = (await model.EmbedAsync([question.Trim()], ct))[0]; + var hits = await index.QueryAsync(model.ModelId, vector, k, ct); + var rows = new List(hits.Count); + foreach (var hit in hits) + { + var doc = await catalog.GetAsync(DocumentKind.Wiki, hit.Slug, ct); + rows.Add(new RankedNote(hit.Slug, doc?.Title ?? hit.Slug, hit.Score)); + } + return rows; + } +} diff --git a/src/Wiki.Application/Status/StatusQuery.cs b/src/Wiki.Application/Status/StatusQuery.cs new file mode 100644 index 0000000..22c9ed4 --- /dev/null +++ b/src/Wiki.Application/Status/StatusQuery.cs @@ -0,0 +1,75 @@ +using Wiki.Application.Sync; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Status; + +public sealed record IndexStatus(string ModelId, int Notes, bool Active); + +public sealed record ServiceStatus( + bool Ready, + string? Commit, + DateTimeOffset? SyncedAt, + double? AgeSeconds, + bool Syncing, + string? LastError, + int Notes, + int RawQueue, + int Outputs, + IReadOnlyList Indexes, + IReadOnlyList Adapters); + +/// +/// Readiness is "the catalog has a commit and the active index has notes". Adapter checks are +/// informational and never gate readiness. +/// +public sealed class StatusQuery( + IDocumentCatalog catalog, + IVectorIndex index, + IReadOnlyList embeddingModels, + IEmbeddingModel activeModel, + SyncCoordinator sync, + IReadOnlyList adapters, + TimeProvider clock) +{ + public async Task IsReadyAsync(CancellationToken ct) + { + var state = await catalog.GetStateAsync(ct); + if (state.Commit is null) return false; + var active = await index.DescribeAsync(activeModel.ModelId, ct); + return active.Count > 0; + } + + public async Task GetAsync(bool includeAdapterChecks, CancellationToken ct) + { + var state = await catalog.GetStateAsync(ct); + var indexes = new List(); + foreach (var m in embeddingModels) + { + var s = await index.DescribeAsync(m.ModelId, ct); + indexes.Add(new IndexStatus(m.ModelId, s.Count, m.ModelId == activeModel.ModelId)); + } + var checks = new List(); + if (includeAdapterChecks) + { + foreach (var a in adapters) + { + try { checks.Add(await a.CheckAsync(ct)); } + catch (Exception e) { checks.Add(new AdapterHealth(a.Name, false, e.Message, TimeSpan.Zero, clock.GetUtcNow())); } + } + } + var now = clock.GetUtcNow(); + return new ServiceStatus( + await IsReadyAsync(ct), + state.Commit, + state.SyncedAt, + state.SyncedAt is { } t ? Math.Round((now - t).TotalSeconds) : null, + sync.Syncing, + sync.LastError, + (await catalog.ListAsync(DocumentKind.Wiki, ct)).Count, + (await catalog.ListAsync(DocumentKind.Raw, ct)).Count, + (await catalog.ListAsync(DocumentKind.Output, ct)).Count, + indexes, + checks); + } +} diff --git a/src/Wiki.Application/Sync/SyncCoordinator.cs b/src/Wiki.Application/Sync/SyncCoordinator.cs new file mode 100644 index 0000000..496550a --- /dev/null +++ b/src/Wiki.Application/Sync/SyncCoordinator.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.Logging; + +namespace Wiki.Application.Sync; + +/// +/// One sync at a time; a caller arriving during a run waits and then runs its own, so a webhook +/// that lands mid-sync is never lost. Keeps the last outcome for status. Failures are recorded +/// and returned as null, never thrown to the trigger. +/// +public sealed class SyncCoordinator(SyncUseCase sync, TimeProvider clock, ILogger log) +{ + private readonly SemaphoreSlim _gate = new(1, 1); + + public bool Syncing { get; private set; } + public SyncSummary? LastSummary { get; private set; } + public string? LastError { get; private set; } + public DateTimeOffset? LastAttemptAt { get; private set; } + + public async Task RunAsync(bool force, CancellationToken ct) + { + await _gate.WaitAsync(ct); + try + { + Syncing = true; + LastAttemptAt = clock.GetUtcNow(); + try + { + LastSummary = await sync.RunAsync(force, ct); + LastError = null; + log.LogInformation("sync done: {Summary}", LastSummary); + return LastSummary; + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + LastError = e.Message.Length > 300 ? e.Message[..300] : e.Message; + log.LogError(e, "sync failed"); + return null; + } + finally + { + Syncing = false; + } + } + finally + { + _gate.Release(); + } + } +} diff --git a/src/Wiki.Application/Sync/SyncUseCase.cs b/src/Wiki.Application/Sync/SyncUseCase.cs new file mode 100644 index 0000000..5068f7a --- /dev/null +++ b/src/Wiki.Application/Sync/SyncUseCase.cs @@ -0,0 +1,119 @@ +using Microsoft.Extensions.Logging; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Application.Sync; + +public sealed record SyncOptions(int EmbedChunk = 20, int EmbedMaxChars = EmbeddingText.DefaultMaxChars); + +public sealed record SyncSummary(string Commit, int Changed, int Removed, int Embedded, int Notes, bool Skipped, TimeSpan Elapsed); + +/// +/// Brings the catalog and the active index to the note store's head. Documents whose content +/// hash is unchanged are neither read nor re-embedded; wiki documents that changed are embedded +/// under every registered embedding model; documents gone from the head are removed. +/// A failure leaves the last good state in place. +/// +public sealed class SyncUseCase( + INoteStore store, + IDocumentCatalog catalog, + IVectorIndex index, + IReadOnlyList embeddingModels, + TimeProvider clock, + ILogger log, + SyncOptions? options = null) +{ + private readonly SyncOptions _options = options ?? new SyncOptions(); + + private static readonly IReadOnlySet AllKinds = + new HashSet { DocumentKind.Wiki, DocumentKind.Raw, DocumentKind.RawProcessed, DocumentKind.Output }; + + private static string Short(string commit) => commit.Length > 8 ? commit[..8] : commit; + + public async Task RunAsync(bool force, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(store); + var started = clock.GetTimestamp(); + var head = await store.GetHeadAsync(ct); + var state = await catalog.GetStateAsync(ct); + var wikiCount = (await catalog.ListAsync(DocumentKind.Wiki, ct)).Count; + + if (!force && state.Commit == head.Commit && wikiCount > 0) + { + await catalog.SetStateAsync(state with { SyncedAt = clock.GetUtcNow() }, ct); + await catalog.FlushAsync(ct); + return new SyncSummary(head.Commit, 0, 0, 0, wikiCount, true, clock.GetElapsedTime(started)); + } + + var listed = await store.ListAsync(head.Commit, AllKinds, ct); + var known = new Dictionary<(DocumentKind, string), DocumentRef>(); + foreach (var kind in AllKinds) + { + foreach (var r in await catalog.ListAsync(kind, ct)) + known[(kind, r.Slug)] = r; + } + + var changed = listed.Where(r => !known.TryGetValue((r.Kind, r.Slug), out var k) || k.ContentHash != r.ContentHash).ToList(); + var listedKeys = listed.Select(r => (r.Kind, r.Slug)).ToHashSet(); + var removed = known.Values.Where(r => !listedKeys.Contains((r.Kind, r.Slug))).ToList(); + log.LogInformation("sync {Commit}: {Listed} documents, {Changed} changed, {Removed} removed", Short(head.Commit), listed.Count, changed.Count, removed.Count); + + var embedded = 0; + foreach (var chunk in changed.Chunk(_options.EmbedChunk)) + { + var documents = new List(chunk.Length); + foreach (var r in chunk) + { + var text = await store.ReadAsync(r, head.Commit, ct); + documents.Add(DocumentParser.Parse(r, text)); + } + await catalog.UpsertAsync(documents, ct); + + var wiki = documents.Where(d => d.Kind == DocumentKind.Wiki).ToList(); + if (wiki.Count == 0) continue; + var texts = wiki.Select(d => EmbeddingText.For(d, _options.EmbedMaxChars)).ToList(); + foreach (var model in embeddingModels) + { + var vectors = await model.EmbedAsync(texts, ct); + var embeddings = wiki.Select((d, i) => new Embedding(model.ModelId, d.Slug, d.ContentHash, vectors[i])).ToList(); + await index.UpsertAsync(embeddings, ct); + } + embedded += wiki.Count; + } + + if (removed.Count > 0) + { + await catalog.RemoveAsync(removed, ct); + var removedWiki = removed.Where(r => r.Kind == DocumentKind.Wiki).Select(r => r.Slug).ToList(); + if (removedWiki.Count > 0) + { + foreach (var model in embeddingModels) + await index.RemoveAsync(model.ModelId, removedWiki, ct); + } + } + + // Index entries that lag the catalog (a model registered later, an interrupted run) are filled here. + foreach (var model in embeddingModels) + { + var indexState = await index.DescribeAsync(model.ModelId, ct); + var wikiDocs = await catalog.ListDocumentsAsync(DocumentKind.Wiki, ct); + var missing = wikiDocs.Where(d => !indexState.ContentHashBySlug.TryGetValue(d.Slug, out var h) || h != d.ContentHash).ToList(); + foreach (var chunk in missing.Chunk(_options.EmbedChunk)) + { + var vectors = await model.EmbedAsync(chunk.Select(d => EmbeddingText.For(d, _options.EmbedMaxChars)).ToList(), ct); + await index.UpsertAsync(chunk.Select((d, i) => new Embedding(model.ModelId, d.Slug, d.ContentHash, vectors[i])).ToList(), ct); + embedded += chunk.Length; + } + var stale = indexState.ContentHashBySlug.Keys.Except(wikiDocs.Select(d => d.Slug)).ToList(); + if (stale.Count > 0) + await index.RemoveAsync(model.ModelId, stale, ct); + } + + await catalog.SetStateAsync(new CatalogState(head.Commit, clock.GetUtcNow()), ct); + await index.FlushAsync(ct); + await catalog.FlushAsync(ct); + var notes = (await catalog.ListAsync(DocumentKind.Wiki, ct)).Count; + return new SyncSummary(head.Commit, changed.Count, removed.Count, embedded, notes, false, clock.GetElapsedTime(started)); + } +} diff --git a/src/Wiki.Application/Wiki.Application.csproj b/src/Wiki.Application/Wiki.Application.csproj index f5a3032..55e999b 100644 --- a/src/Wiki.Application/Wiki.Application.csproj +++ b/src/Wiki.Application/Wiki.Application.csproj @@ -4,6 +4,10 @@ + + + + net10.0 enable diff --git a/src/Wiki.Core/Model/Answer.cs b/src/Wiki.Core/Model/Answer.cs new file mode 100644 index 0000000..0021c4b --- /dev/null +++ b/src/Wiki.Core/Model/Answer.cs @@ -0,0 +1,20 @@ +namespace Wiki.Core.Model; + +/// A paragraph handed to the answer model as evidence, with the judge's probability that it answers. +public sealed record EvidenceParagraph(Paragraph Paragraph, string NoteTitle, double Probability); + +/// Everything the answer model is allowed to see. Not the whole context: the selected evidence and the judgements. +public sealed record AnswerRequest( + string Question, + IReadOnlyList Evidence, + IReadOnlyDictionary Judgements, + double Covered); + +public abstract record AnswerChunk +{ + /// A piece of the answer text, in order. + public sealed record Text(string Delta) : AnswerChunk; + + /// The end of the answer, with what it cost. + public sealed record Done(string ModelId, long InputTokens, long OutputTokens, TimeSpan Elapsed) : AnswerChunk; +} diff --git a/src/Wiki.Core/Model/Document.cs b/src/Wiki.Core/Model/Document.cs new file mode 100644 index 0000000..0ae71d6 --- /dev/null +++ b/src/Wiki.Core/Model/Document.cs @@ -0,0 +1,76 @@ +namespace Wiki.Core.Model; + +/// Where in the vault a document lives. Kinds are disjoint sets of paths. +public enum DocumentKind +{ + /// wiki/*.md — the retrieval corpus, embedded. + Wiki, + /// raw/*.md — sources waiting for distillation. + Raw, + /// raw/_processed/*.md — sources distillation has finished with; provenance. + RawProcessed, + /// output/*.md — synthesised answers, audit and consolidation reports. + Output, +} + +/// A listing entry: enough to know whether a document changed without reading it. +public sealed record DocumentRef(DocumentKind Kind, string Slug, string Path, string ContentHash); + +/// A parsed vault document. Frontmatter fields the vault uses are lifted into properties; the rest stay in . +public sealed record Document( + DocumentKind Kind, + string Slug, + string Path, + string ContentHash, + string Title, + string? Type, + IReadOnlyList Tags, + IReadOnlyList Aliases, + IReadOnlyList Related, + IReadOnlyList Sources, + DateOnly? Updated, + IReadOnlyDictionary Frontmatter, + string Body, + int BodyStartLine) +{ + public DocumentRef Ref => new(Kind, Slug, Path, ContentHash); +} + +/// A frontmatter value: a scalar or a list of scalars. The vault uses nothing deeper. +public sealed record FrontmatterValue(string? Scalar, IReadOnlyList? List) +{ + public static FrontmatterValue Of(string scalar) => new(scalar, null); + public static FrontmatterValue Of(IReadOnlyList list) => new(null, list); + public bool IsList => List is not null; + + /// The value as a list: a list as is, a scalar as a one-element list, an empty scalar as empty. + public IReadOnlyList AsList() => + List ?? (string.IsNullOrWhiteSpace(Scalar) ? [] : [Scalar!]); +} + +/// Part of a document body addressable in typed questions: slug#pN. +public sealed record Paragraph(string Id, string Slug, string Heading, string Text); + +/// An outgoing reference from one document to another, by target slug. +public sealed record Link(string FromSlug, string ToSlug, LinkKind Kind, int? Line); + +public enum LinkKind +{ + /// [[target]] in the body. + WikiLink, + /// An entry in the related: frontmatter list. + Related, +} + +/// A caller identity for attribution. Resolved by the host's authentication, carried into use cases. +public sealed record Identity(string Name, IdentityKind Kind) +{ + public static readonly Identity Anonymous = new("anonymous", IdentityKind.Anonymous); +} + +public enum IdentityKind +{ + Anonymous, + Human, + Machine, +} diff --git a/src/Wiki.Core/Model/Health.cs b/src/Wiki.Core/Model/Health.cs new file mode 100644 index 0000000..7219fa9 --- /dev/null +++ b/src/Wiki.Core/Model/Health.cs @@ -0,0 +1,4 @@ +namespace Wiki.Core.Model; + +/// Result of an adapter's self check. Reported in status, never used for readiness. +public sealed record AdapterHealth(string Adapter, bool Ok, string? Detail, TimeSpan Latency, DateTimeOffset CheckedAt); diff --git a/src/Wiki.Core/Model/Judgement.cs b/src/Wiki.Core/Model/Judgement.cs new file mode 100644 index 0000000..5d5fea9 --- /dev/null +++ b/src/Wiki.Core/Model/Judgement.cs @@ -0,0 +1,36 @@ +namespace Wiki.Core.Model; + +/// +/// A typed question over a state. Three primitives, as TypeSafe's System One defines them; +/// any judge adapter has to answer all three. +/// +public abstract record TypedQuestion(string Instructions) +{ + /// A yes/no judgement with a probability. Criteria describe both outcomes. + public sealed record Noul(string Instructions, string TrueCriterion, string FalseCriterion) : TypedQuestion(Instructions); + + /// One option out of many, with a probability per option. Cap is the judge's concern. + public sealed record Choice(string Instructions, IReadOnlyDictionary Options) : TypedQuestion(Instructions); + + /// A graded judgement in [0, 1]. Criteria describe the ends of the scale. + public sealed record Score(string Instructions, string Criteria) : TypedQuestion(Instructions); +} + +/// The state a judge sees plus the questions asked about it, one request. +public sealed record JudgeRequest(object State, IReadOnlyDictionary Questions); + +public abstract record Judgement +{ + public sealed record Noul(double Probability) : Judgement; + public sealed record Choice(string Selected, IReadOnlyDictionary Probabilities, double Confidence) : Judgement; + public sealed record Score(double Value) : Judgement; +} + +public sealed record JudgeUsage(int Requests, long InputTokens, long OutputTokens, TimeSpan Elapsed) +{ + public static readonly JudgeUsage None = new(0, 0, 0, TimeSpan.Zero); + public JudgeUsage Add(JudgeUsage other) => + new(Requests + other.Requests, InputTokens + other.InputTokens, OutputTokens + other.OutputTokens, Elapsed + other.Elapsed); +} + +public sealed record JudgeResult(IReadOnlyDictionary Answers, JudgeUsage Usage); diff --git a/src/Wiki.Core/Model/RawSource.cs b/src/Wiki.Core/Model/RawSource.cs new file mode 100644 index 0000000..c72141f --- /dev/null +++ b/src/Wiki.Core/Model/RawSource.cs @@ -0,0 +1,29 @@ +namespace Wiki.Core.Model; + +/// +/// A fact an agent deposits. Becomes one file in raw/ with provenance frontmatter. +/// There is deliberately no field for a hypothesis: the shape enforces the raw rule. +/// +public sealed record RawSource( + string Title, + string Source, + string Content, + DateTimeOffset ObservedAt, + IReadOnlyList Tags) +{ + public const int MinimumContentLength = 40; + + /// Validation errors, empty when the fact may be stored. + public IReadOnlyList Validate() + { + var errors = new List(); + if (string.IsNullOrWhiteSpace(Title)) errors.Add("title is required"); + if (string.IsNullOrWhiteSpace(Source)) errors.Add("source is required: where the fact was observed or quoted from"); + if (Content is null || Content.Trim().Length < MinimumContentLength) + errors.Add($"content must be at least {MinimumContentLength} characters of observation"); + return errors; + } +} + +/// What the fact sink returns once the source is stored. +public sealed record FactReceipt(string Path, string Commit, DateTimeOffset StoredAt); diff --git a/src/Wiki.Core/Model/Retrieval.cs b/src/Wiki.Core/Model/Retrieval.cs new file mode 100644 index 0000000..ff61934 --- /dev/null +++ b/src/Wiki.Core/Model/Retrieval.cs @@ -0,0 +1,16 @@ +namespace Wiki.Core.Model; + +/// A vector for one document under one embedding model. +public sealed record Embedding(string ModelId, string Slug, string ContentHash, float[] Vector); + +/// One result of a vector query. +public sealed record RankedHit(string Slug, float Score); + +/// What an index holds for one model: which documents at which content hash. +public sealed record IndexState(string ModelId, IReadOnlyDictionary ContentHashBySlug) +{ + public int Count => ContentHashBySlug.Count; +} + +/// Head of the note store: the commit the listing is taken from. +public sealed record StoreHead(string Commit, DateTimeOffset? CommittedAt); diff --git a/src/Wiki.Core/Parsing/DocumentParser.cs b/src/Wiki.Core/Parsing/DocumentParser.cs new file mode 100644 index 0000000..4c8db7e --- /dev/null +++ b/src/Wiki.Core/Parsing/DocumentParser.cs @@ -0,0 +1,72 @@ +using System.Globalization; +using Wiki.Core.Model; + +namespace Wiki.Core.Parsing; + +/// Turns a vault file into a . Kind and slug come from the path. +public static class DocumentParser +{ + public static Document Parse(DocumentRef reference, string text) + { + var (fields, body, bodyStart) = Frontmatter.Parse(text); + var title = Field(fields, "title")?.Scalar?.Trim(); + if (string.IsNullOrEmpty(title)) title = reference.Slug; + return new Document( + reference.Kind, + reference.Slug, + reference.Path, + reference.ContentHash, + title, + Field(fields, "type")?.Scalar?.Trim(), + ListField(fields, "tags"), + ListField(fields, "aliases"), + ListField(fields, "related"), + ListField(fields, "sources"), + ParseDate(Field(fields, "updated")?.Scalar), + fields, + body, + bodyStart); + } + + /// Kind and slug for a repository path, or null when the path is not a vault document. + public static (DocumentKind Kind, string Slug)? Classify(string path) + { + if (!path.EndsWith(".md", StringComparison.Ordinal)) return null; + var slash = path.LastIndexOf('/'); + var dir = slash < 0 ? string.Empty : path[..(slash + 1)]; + var name = path[(slash + 1)..^3]; + if (name.Length == 0 || name.StartsWith('_')) return null; // _index.md and friends + return dir switch + { + "wiki/" => (DocumentKind.Wiki, name), + "raw/" => (DocumentKind.Raw, name), + "raw/_processed/" => (DocumentKind.RawProcessed, name), + "output/" => (DocumentKind.Output, name), + _ => null, + }; + } + + public static string PathFor(DocumentKind kind, string slug) => kind switch + { + DocumentKind.Wiki => $"wiki/{slug}.md", + DocumentKind.Raw => $"raw/{slug}.md", + DocumentKind.RawProcessed => $"raw/_processed/{slug}.md", + DocumentKind.Output => $"output/{slug}.md", + _ => throw new ArgumentOutOfRangeException(nameof(kind)), + }; + + private static FrontmatterValue? Field(IReadOnlyDictionary fields, string key) => + fields.TryGetValue(key, out var v) ? v : null; + + private static IReadOnlyList ListField(IReadOnlyDictionary fields, string key) => + Field(fields, key)?.AsList() ?? []; + + public static DateOnly? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return null; + var s = value.Trim(); + if (s.Length >= 10 && DateOnly.TryParseExact(s[..10], "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var d)) + return d; + return null; + } +} diff --git a/src/Wiki.Core/Parsing/EmbeddingText.cs b/src/Wiki.Core/Parsing/EmbeddingText.cs new file mode 100644 index 0000000..d6c0ca2 --- /dev/null +++ b/src/Wiki.Core/Parsing/EmbeddingText.cs @@ -0,0 +1,50 @@ +using Wiki.Core.Model; + +namespace Wiki.Core.Parsing; + +/// +/// The text a note is embedded from: slug words, title, aliases, tags and the first paragraph +/// after the H1, capped. Same composition as the Python wiki-index so the two rank alike. +/// +public static class EmbeddingText +{ + public const int DefaultMaxChars = 1500; + + public static string For(Document document, int maxChars = DefaultMaxChars) + { + var parts = new[] + { + document.Slug.Replace('-', ' '), + document.Title, + string.Join(' ', document.Aliases), + string.Join(' ', document.Tags), + FirstParagraph(document.Body), + }; + var text = string.Join(' ', parts); + return text.Length > maxChars ? text[..maxChars] : text; + } + + /// The first non-heading paragraph after the H1; empty when there is no H1. + public static string FirstParagraph(string body) + { + var para = new List(); + var seenH1 = false; + foreach (var raw in body.Split('\n')) + { + var line = raw.TrimEnd('\r'); + if (line.StartsWith("# ", StringComparison.Ordinal)) + { + seenH1 = true; + continue; + } + if (!seenH1) continue; + if (line.Trim().Length == 0 || line.StartsWith('#')) + { + if (para.Count > 0) break; + continue; + } + para.Add(line.Trim()); + } + return string.Join(' ', para); + } +} diff --git a/src/Wiki.Core/Parsing/Frontmatter.cs b/src/Wiki.Core/Parsing/Frontmatter.cs new file mode 100644 index 0000000..4a16b9e --- /dev/null +++ b/src/Wiki.Core/Parsing/Frontmatter.cs @@ -0,0 +1,134 @@ +using System.Text; +using Wiki.Core.Model; + +namespace Wiki.Core.Parsing; + +/// +/// Reads the YAML subset the vault uses: key: scalar, key: [a, "b, c", 'd'], +/// block lists of - item, quoted scalars, and nothing nested deeper. Tolerant: a line +/// it cannot read is skipped, never fatal, because a note with odd frontmatter is still a note. +/// +public static class Frontmatter +{ + public static (IReadOnlyDictionary Fields, string Body, int BodyStartLine) Parse(string text) + { + var fields = new Dictionary(StringComparer.Ordinal); + if (!text.StartsWith("---", StringComparison.Ordinal)) + return (fields, text, 1); + + var lines = text.Split('\n'); + if (lines[0].TrimEnd('\r').Trim() != "---") + return (fields, text, 1); + + var end = -1; + for (var i = 1; i < lines.Length; i++) + { + if (lines[i].TrimEnd('\r').Trim() == "---") { end = i; break; } + } + if (end < 0) + return (fields, text, 1); + + string? pendingKey = null; + List? pendingList = null; + for (var i = 1; i < end; i++) + { + var line = lines[i].TrimEnd('\r'); + if (string.IsNullOrWhiteSpace(line) || line.TrimStart().StartsWith('#')) + continue; + + if (pendingKey is not null && line.TrimStart().StartsWith("- ", StringComparison.Ordinal)) + { + pendingList ??= []; + pendingList.Add(Unquote(line.TrimStart()[2..].Trim())); + continue; + } + if (pendingKey is not null) + { + fields[pendingKey] = pendingList is null ? FrontmatterValue.Of(string.Empty) : FrontmatterValue.Of(pendingList); + pendingKey = null; + pendingList = null; + } + + var colon = IndexOfKeyColon(line); + if (colon < 0) + continue; + var key = line[..colon].Trim(); + var value = line[(colon + 1)..].Trim(); + if (key.Length == 0) + continue; + + if (value.Length == 0) + { + pendingKey = key; // a block list may follow; otherwise an empty scalar + continue; + } + if (value.StartsWith('[') && value.EndsWith(']')) + { + fields[key] = FrontmatterValue.Of(SplitFlowList(value[1..^1])); + continue; + } + fields[key] = FrontmatterValue.Of(Unquote(value)); + } + if (pendingKey is not null) + fields[pendingKey] = pendingList is null ? FrontmatterValue.Of(string.Empty) : FrontmatterValue.Of(pendingList); + + var body = string.Join('\n', lines.Skip(end + 1)); + return (fields, body, end + 2); + } + + private static int IndexOfKeyColon(string line) + { + // A key is [A-Za-z_][\w-]* at column 0 followed by ':' and a space or end of line. + var i = 0; + if (i >= line.Length || !(char.IsLetter(line[i]) || line[i] == '_')) return -1; + while (i < line.Length && (char.IsLetterOrDigit(line[i]) || line[i] == '_' || line[i] == '-')) i++; + if (i >= line.Length || line[i] != ':') return -1; + if (i + 1 < line.Length && line[i + 1] != ' ' && line[i + 1] != '\t') return -1; + return i; + } + + /// Splits a, "b, c", 'd' respecting quotes; sources lists carry commas inside quotes. + public static IReadOnlyList SplitFlowList(string inner) + { + var items = new List(); + var current = new StringBuilder(); + char? quote = null; + foreach (var ch in inner) + { + if (quote is null && (ch == '"' || ch == '\'')) + { + quote = ch; + current.Append(ch); + continue; + } + if (quote is not null && ch == quote) + { + quote = null; + current.Append(ch); + continue; + } + if (quote is null && ch == ',') + { + Push(items, current); + continue; + } + current.Append(ch); + } + Push(items, current); + return items; + + static void Push(List items, StringBuilder sb) + { + var s = Unquote(sb.ToString().Trim()); + sb.Clear(); + if (s.Length > 0) items.Add(s); + } + } + + public static string Unquote(string value) + { + if (value.Length >= 2 && ((value[0] == '"' && value[^1] == '"') || (value[0] == '\'' && value[^1] == '\''))) + return value[1..^1]; + return value; + } +} diff --git a/src/Wiki.Core/Parsing/ParagraphChunker.cs b/src/Wiki.Core/Parsing/ParagraphChunker.cs new file mode 100644 index 0000000..8e90b2d --- /dev/null +++ b/src/Wiki.Core/Parsing/ParagraphChunker.cs @@ -0,0 +1,63 @@ +using System.Text; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Core.Parsing; + +/// +/// The vault's paragraph splitter, as wiki_ask.py does it: blank-line separated blocks, +/// headings set the block's heading and are not blocks themselves, fenced code is one block. +/// Ids are slug#pN, counted from 1 in document order. +/// +public sealed class ParagraphChunker : IChunker +{ + public IReadOnlyList Split(Document document) + { + var paragraphs = new List(); + var heading = document.Title; + var block = new StringBuilder(); + var fenced = false; + + void Flush() + { + var text = block.ToString().Trim(); + block.Clear(); + if (text.Length == 0) return; + var n = paragraphs.Count + 1; + paragraphs.Add(new Paragraph($"{document.Slug}#p{n}", document.Slug, heading, text)); + } + + foreach (var raw in document.Body.Split('\n')) + { + var line = raw.TrimEnd('\r'); + var trimmed = line.TrimStart(); + if (trimmed.StartsWith("```", StringComparison.Ordinal) || trimmed.StartsWith("~~~", StringComparison.Ordinal)) + { + if (!fenced) Flush(); + fenced = !fenced; + block.Append(line).Append('\n'); + if (!fenced) Flush(); + continue; + } + if (fenced) + { + block.Append(line).Append('\n'); + continue; + } + if (trimmed.StartsWith('#')) + { + Flush(); + heading = trimmed.TrimStart('#').Trim(); + continue; + } + if (string.IsNullOrWhiteSpace(line)) + { + Flush(); + continue; + } + block.Append(line).Append('\n'); + } + Flush(); + return paragraphs; + } +} diff --git a/src/Wiki.Core/Parsing/WikiLinks.cs b/src/Wiki.Core/Parsing/WikiLinks.cs new file mode 100644 index 0000000..5d514fb --- /dev/null +++ b/src/Wiki.Core/Parsing/WikiLinks.cs @@ -0,0 +1,52 @@ +using System.Text.RegularExpressions; +using Wiki.Core.Model; + +namespace Wiki.Core.Parsing; + +/// Outgoing references of a document: [[links]] in the body outside code, plus the related list. +public static partial class WikiLinks +{ + [GeneratedRegex(@"\[\[([^\]\[|#]+)(?:#[^\]\[|]*)?(?:\|[^\]\[]*)?\]\]")] + private static partial Regex LinkPattern(); + + [GeneratedRegex(@"^[A-Za-z0-9][A-Za-z0-9._-]*$")] + private static partial Regex SlugPattern(); + + [GeneratedRegex("`[^`]*`")] + private static partial Regex InlineCode(); + + public static bool IsSlug(string value) => SlugPattern().IsMatch(value); + + public static IReadOnlyList Extract(Document document) + { + var links = new List(); + foreach (var target in document.Related) + { + if (SlugPattern().IsMatch(target)) + links.Add(new Link(document.Slug, target, LinkKind.Related, null)); + } + + var fenced = false; + var lineNo = document.BodyStartLine - 1; + foreach (var raw in document.Body.Split('\n')) + { + lineNo++; + var line = raw.TrimEnd('\r'); + var trimmed = line.TrimStart(); + if (trimmed.StartsWith("```", StringComparison.Ordinal) || trimmed.StartsWith("~~~", StringComparison.Ordinal)) + { + fenced = !fenced; + continue; + } + if (fenced) continue; + var stripped = InlineCode().Replace(line, string.Empty); + foreach (Match m in LinkPattern().Matches(stripped)) + { + var target = m.Groups[1].Value.Trim(); + if (SlugPattern().IsMatch(target)) + links.Add(new Link(document.Slug, target, LinkKind.WikiLink, lineNo)); + } + } + return links; + } +} diff --git a/src/Wiki.Core/Ports/Ports.cs b/src/Wiki.Core/Ports/Ports.cs new file mode 100644 index 0000000..1c0a039 --- /dev/null +++ b/src/Wiki.Core/Ports/Ports.cs @@ -0,0 +1,90 @@ +using Wiki.Core.Model; + +namespace Wiki.Core.Ports; + +/// Anything that can say whether it is reachable and working. Every adapter implements it. +public interface IAdapterHealth +{ + string Name { get; } + Task CheckAsync(CancellationToken ct); +} + +/// Where the vault's documents are read from. +public interface INoteStore : IAdapterHealth +{ + /// The current head; a listing is always taken at a head. + Task GetHeadAsync(CancellationToken ct); + + /// Every document of the given kinds at that head, with a content hash that changes iff the content changes. + Task> ListAsync(string commit, IReadOnlySet kinds, CancellationToken ct); + + /// The document's text at that head. + Task ReadAsync(DocumentRef reference, string commit, CancellationToken ct); +} + +/// Where a deposited fact goes: one new raw source, never an edit of an existing file. +public interface IFactSink : IAdapterHealth +{ + Task AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct); +} + +/// Text to vector. One instance per configured model. +public interface IEmbeddingModel : IAdapterHealth +{ + string ModelId { get; } + int Dimension { get; } + Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct); +} + +/// Vectors per (model, document) with top-k query. Models are namespaces; nothing is shared across them. +public interface IVectorIndex : IAdapterHealth +{ + Task DescribeAsync(string modelId, CancellationToken ct); + Task UpsertAsync(IReadOnlyList embeddings, CancellationToken ct); + Task RemoveAsync(string modelId, IReadOnlyList slugs, CancellationToken ct); + Task> QueryAsync(string modelId, float[] query, int k, CancellationToken ct); + /// Persist whatever the adapter keeps in memory. A no-op for adapters that write through. + Task FlushAsync(CancellationToken ct); +} + +/// The synced documents, queryable: bodies, frontmatter and the link graph. Filled by the sync, read by everything else. +public interface IDocumentCatalog : IAdapterHealth +{ + Task GetStateAsync(CancellationToken ct); + Task SetStateAsync(CatalogState state, CancellationToken ct); + Task UpsertAsync(IReadOnlyList documents, CancellationToken ct); + Task RemoveAsync(IReadOnlyList references, CancellationToken ct); + Task GetAsync(DocumentKind kind, string slug, CancellationToken ct); + Task> ListAsync(DocumentKind kind, CancellationToken ct); + Task> ListDocumentsAsync(DocumentKind kind, CancellationToken ct); + /// Wiki documents that reference the slug, by wikilink or related entry. + Task> BacklinksAsync(string slug, CancellationToken ct); + Task> OutgoingLinksAsync(string slug, CancellationToken ct); + Task> TagCountsAsync(CancellationToken ct); + Task FlushAsync(CancellationToken ct); +} + +/// Which head the catalog reflects and when it got there. +public sealed record CatalogState(string? Commit, DateTimeOffset? SyncedAt); + +/// Splits a document body into addressable paragraphs for typed questions. +public interface IChunker +{ + IReadOnlyList Split(Document document); +} + +/// Typed questions over a state. Returns probabilities, never prose. +public interface IJudge : IAdapterHealth +{ + string ModelId { get; } + /// The largest number of options one Choice may carry, or null for no cap. + int? MaxChoiceOptions { get; } + Task JudgeAsync(JudgeRequest request, CancellationToken ct); +} + +/// Prose from a question and its evidence. Sees only what the request carries. +public interface IAnswerModel : IAdapterHealth +{ + string ModelId { get; } + IAsyncEnumerable AnswerAsync(AnswerRequest request, CancellationToken ct); +} diff --git a/src/Wiki.Host/Api/ApiV1.cs b/src/Wiki.Host/Api/ApiV1.cs new file mode 100644 index 0000000..546db0f --- /dev/null +++ b/src/Wiki.Host/Api/ApiV1.cs @@ -0,0 +1,24 @@ +using Wiki.Application.Status; +using Wiki.Application.Sync; + +namespace Wiki.Host.Api; + +/// /api/v1: the read API grows here phase by phase. Phase 1 exposes status and a manual sync. +public static class ApiV1 +{ + public static void MapApiV1(this WebApplication app) + { + var v1 = app.MapGroup("/api/v1"); + + v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) => + Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous(); + + v1.MapPost("/sync", async (SyncCoordinator sync, bool? force, CancellationToken ct) => + { + var summary = await sync.RunAsync(force ?? false, ct); + return summary is null + ? Results.Json(new { error = sync.LastError }, statusCode: StatusCodes.Status502BadGateway) + : Results.Ok(summary); + }).RequireAuthorization(Auth.Policies.Write); + } +} diff --git a/src/Wiki.Host/Api/HealthEndpoints.cs b/src/Wiki.Host/Api/HealthEndpoints.cs new file mode 100644 index 0000000..b95ce56 --- /dev/null +++ b/src/Wiki.Host/Api/HealthEndpoints.cs @@ -0,0 +1,22 @@ +using Wiki.Application.Status; + +namespace Wiki.Host.Api; + +public static class HealthEndpoints +{ + /// + /// /health: the process serves, always 200 (liveness). + /// /ready: catalog synced once and the active index loaded (readiness and startup). + /// Adapter checks belong to /api/v1/status and never gate readiness. + /// + public static void MapHealthEndpoints(this WebApplication app) + { + app.MapGet("/health", () => Results.Ok(new { ok = true })).AllowAnonymous(); + app.MapGet("/ready", async (StatusQuery status, CancellationToken ct) => + { + var ready = await status.IsReadyAsync(ct); + var body = await status.GetAsync(includeAdapterChecks: false, ct); + return ready ? Results.Ok(body) : Results.Json(body, statusCode: StatusCodes.Status503ServiceUnavailable); + }).AllowAnonymous(); + } +} diff --git a/src/Wiki.Host/Api/IndexEndpoints.cs b/src/Wiki.Host/Api/IndexEndpoints.cs new file mode 100644 index 0000000..fe821b4 --- /dev/null +++ b/src/Wiki.Host/Api/IndexEndpoints.cs @@ -0,0 +1,77 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Wiki.Application.Retrieval; +using Wiki.Application.Status; +using Wiki.Core.Ports; +using Wiki.Host.Composition; + +namespace Wiki.Host.Api; + +public sealed record RankRequest(string? Query, string? Question, int? K, string? Model); + +/// +/// The endpoints the Python wiki-index served, kept shape-compatible so bin/wiki_rank.py works +/// unchanged during the migration: POST /rank {query, k} → {rows: [{slug, title, score}], index}, +/// POST /webhook (Gitea push, HMAC-SHA256 in X-Gitea-Signature). +/// +public static class IndexEndpoints +{ + public static void MapIndexEndpoints(this WebApplication app) + { + app.MapPost("/rank", RankAsync).AllowAnonymous(); + app.MapPost("/api/v1/rank", RankAsync).AllowAnonymous(); + + app.MapPost("/webhook", async (HttpRequest request, IConfiguration configuration, WikiOptions options, SyncBackgroundService sync, CancellationToken ct) => + { + var secret = configuration["Wiki:Webhook:Secret"]; + if (string.IsNullOrEmpty(secret)) + return Results.Json(new { error = "webhook secret not configured" }, statusCode: StatusCodes.Status503ServiceUnavailable); + using var ms = new MemoryStream(); + await request.Body.CopyToAsync(ms, ct); + var body = ms.ToArray(); + var expected = Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(secret), body)); + var presented = (request.Headers["X-Gitea-Signature"].FirstOrDefault() ?? string.Empty).Trim(); + if (!CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(expected), Encoding.ASCII.GetBytes(presented))) + return Results.Json(new { error = "bad signature" }, statusCode: StatusCodes.Status403Forbidden); + + string? @ref = null; + try + { + using var doc = JsonDocument.Parse(body.Length == 0 ? "{}"u8.ToArray() : body); + if (doc.RootElement.TryGetProperty("ref", out var r)) @ref = r.GetString(); + } + catch (JsonException) + { + return Results.BadRequest(new { error = "bad json" }); + } + if (!string.IsNullOrEmpty(@ref) && @ref != $"refs/heads/{options.NoteStore.Gitea.Branch}") + return Results.Ok(new { queued = false, @ref }); + sync.Trigger(); + return Results.Ok(new { queued = true }); + }).AllowAnonymous(); + } + + private static async Task RankAsync(RankRequest body, RankUseCase rank, StatusQuery status, IServiceProvider services, IEmbeddingModel active, CancellationToken ct) + { + var question = (body.Query ?? body.Question ?? string.Empty).Trim(); + if (question.Length == 0) return Results.BadRequest(new { error = "query required" }); + var model = active; + if (!string.IsNullOrEmpty(body.Model)) + { + model = services.GetKeyedService(body.Model); + if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" }); + } + try + { + var rows = await rank.RankAsync(model, question, body.K ?? 60, ct); + var index = await status.GetAsync(includeAdapterChecks: false, ct); + return Results.Ok(new { rows = rows.Select(r => new { r.Slug, r.Title, r.Score }), model = model.ModelId, index }); + } + catch (HttpRequestException e) + { + return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway); + } + } +} diff --git a/src/Wiki.Host/Auth/ApiKeyAuthenticationHandler.cs b/src/Wiki.Host/Auth/ApiKeyAuthenticationHandler.cs new file mode 100644 index 0000000..c44d491 --- /dev/null +++ b/src/Wiki.Host/Auth/ApiKeyAuthenticationHandler.cs @@ -0,0 +1,54 @@ +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Options; + +namespace Wiki.Host.Auth; + +/// +/// Machine callers present `X-Api-Key: ` or `Authorization: Bearer `. Keys are named, +/// so a write carries the identity behind it: Auth:ApiKeys:{name} = key. Fail closed: no keys +/// configured means no key authenticates. +/// +public sealed class ApiKeyAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + IConfiguration configuration) + : AuthenticationHandler(options, logger, encoder) +{ + public const string SchemeName = "ApiKey"; + + protected override Task HandleAuthenticateAsync() + { + var presented = Request.Headers["X-Api-Key"].FirstOrDefault(); + if (presented is null) + { + var authorization = Request.Headers.Authorization.FirstOrDefault(); + if (authorization?.StartsWith("Bearer ", StringComparison.Ordinal) == true) + presented = authorization["Bearer ".Length..]; + } + if (presented is null) + return Task.FromResult(AuthenticateResult.NoResult()); + + var presentedBytes = Encoding.UTF8.GetBytes(presented); + foreach (var child in configuration.GetSection("Auth:ApiKeys").GetChildren()) + { + var configured = child.Value; + if (string.IsNullOrEmpty(configured)) continue; + if (CryptographicOperations.FixedTimeEquals(presentedBytes, Encoding.UTF8.GetBytes(configured))) + { + var identity = new ClaimsIdentity([new Claim(ClaimTypes.Name, child.Key), new Claim(Claims.Kind, "machine")], Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(new ClaimsPrincipal(identity), Scheme.Name))); + } + } + return Task.FromResult(AuthenticateResult.Fail("invalid API key")); + } +} + +public static class Policies +{ + public const string Write = "write"; +} diff --git a/src/Wiki.Host/Auth/ProxyHeaderAuthenticationHandler.cs b/src/Wiki.Host/Auth/ProxyHeaderAuthenticationHandler.cs new file mode 100644 index 0000000..e1a503b --- /dev/null +++ b/src/Wiki.Host/Auth/ProxyHeaderAuthenticationHandler.cs @@ -0,0 +1,71 @@ +using System.Net; +using System.Security.Claims; +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Options; + +namespace Wiki.Host.Auth; + +/// +/// Humans arrive through the Authentik proxy outpost, which sets X-Authentik-Username after +/// validating the session. The header is trusted only from the peers listed in +/// Auth:ReverseProxy:TrustedProxies; with none configured the scheme authenticates nobody. +/// +public sealed class ProxyHeaderAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + IConfiguration configuration) + : AuthenticationHandler(options, logger, encoder) +{ + public const string SchemeName = "ProxyHeader"; + public const string DefaultHeaderName = "X-Authentik-Username"; + + protected override Task HandleAuthenticateAsync() + { + var headerName = configuration["Auth:ReverseProxy:HeaderName"]; + if (string.IsNullOrWhiteSpace(headerName)) headerName = DefaultHeaderName; + + var user = Request.Headers[headerName].FirstOrDefault(); + if (string.IsNullOrWhiteSpace(user)) + return Task.FromResult(AuthenticateResult.NoResult()); + + var peer = Context.Connection.RemoteIpAddress; + if (peer is null || !IsTrustedProxy(peer)) + { + Logger.LogWarning("ignoring {Header} from untrusted peer {Peer}", headerName, peer); + return Task.FromResult(AuthenticateResult.NoResult()); + } + + var identity = new ClaimsIdentity([new Claim(ClaimTypes.Name, user), new Claim(Claims.Kind, "human")], Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(new ClaimsPrincipal(identity), Scheme.Name))); + } + + protected override Task HandleChallengeAsync(AuthenticationProperties properties) + { + Response.StatusCode = StatusCodes.Status401Unauthorized; + return Task.CompletedTask; + } + + private bool IsTrustedProxy(IPAddress peer) + { + if (peer.IsIPv4MappedToIPv6) peer = peer.MapToIPv4(); + var trusted = configuration.GetSection("Auth:ReverseProxy:TrustedProxies").Get() ?? []; + foreach (var entry in trusted) + { + if (IPAddress.TryParse(entry, out var single)) + { + if (single.Equals(peer)) return true; + continue; + } + if (IPNetwork.TryParse(entry, out var network) && network.Contains(peer)) + return true; + } + return false; + } +} + +public static class Claims +{ + public const string Kind = "wiki:kind"; +} diff --git a/src/Wiki.Host/Components/App.razor b/src/Wiki.Host/Components/App.razor index fd5bc2b..d0935ad 100644 --- a/src/Wiki.Host/Components/App.razor +++ b/src/Wiki.Host/Components/App.razor @@ -1,4 +1,4 @@ - + @@ -6,6 +6,7 @@ + @@ -16,6 +17,7 @@ + diff --git a/src/Wiki.Host/Components/Layout/MainLayout.razor b/src/Wiki.Host/Components/Layout/MainLayout.razor index 96fbbe6..35310bf 100644 --- a/src/Wiki.Host/Components/Layout/MainLayout.razor +++ b/src/Wiki.Host/Components/Layout/MainLayout.razor @@ -1,6 +1,22 @@ -@inherits LayoutComponentBase +@inherits LayoutComponentBase -@Body + + + + + + + + Wiki + + Status + + + + @Body + + +
An unhandled error has occurred. diff --git a/src/Wiki.Host/Components/Pages/Home.razor b/src/Wiki.Host/Components/Pages/Home.razor index 9001e0b..eb503f3 100644 --- a/src/Wiki.Host/Components/Pages/Home.razor +++ b/src/Wiki.Host/Components/Pages/Home.razor @@ -1,7 +1,78 @@ -@page "/" +@page "/" +@rendermode InteractiveServer +@using Wiki.Application.Status +@inject StatusQuery Status -Home +Wiki status -

Hello, world!

+Status -Welcome to your new app. +@if (_status is null) +{ + +} +else +{ + + + + Index + + + Ready@(_status.Ready ? "yes" : "no") + Commit@(_status.Commit is { Length: > 8 } c ? c[..8] : _status.Commit) + Synced@(_status.SyncedAt?.ToString("u") ?? "never") (@(_status.AgeSeconds?.ToString("0") ?? "-") s ago) + Syncing@(_status.Syncing ? "yes" : "no") + Notes@_status.Notes + Raw queue@_status.RawQueue + Outputs@_status.Outputs + @if (_status.LastError is not null) + { + Last error@_status.LastError + } + + + + + + + Embedding models + + + @foreach (var i in _status.Indexes) + { + @i.ModelId @(i.Active ? "(active)" : "")@i.Notes vectors + } + + + + + + + Adapters + + + @foreach (var a in _status.Adapters) + { + + @a.Adapter + @(a.Ok ? "ok" : "failing") + @a.Latency.TotalMilliseconds.ToString("0") ms + @a.Detail + + } + + + + + +} + +@code { + private ServiceStatus? _status; + + protected override async Task OnInitializedAsync() + { + _status = await Status.GetAsync(includeAdapterChecks: true, CancellationToken.None); + } +} diff --git a/src/Wiki.Host/Components/_Imports.razor b/src/Wiki.Host/Components/_Imports.razor index 1b19ea5..5f50cd4 100644 --- a/src/Wiki.Host/Components/_Imports.razor +++ b/src/Wiki.Host/Components/_Imports.razor @@ -1,4 +1,4 @@ -@using System.Net.Http +@using System.Net.Http @using System.Net.Http.Json @using Microsoft.AspNetCore.Components.Forms @using Microsoft.AspNetCore.Components.Routing @@ -6,6 +6,7 @@ @using static Microsoft.AspNetCore.Components.Web.RenderMode @using Microsoft.AspNetCore.Components.Web.Virtualization @using Microsoft.JSInterop +@using MudBlazor @using Wiki.Host @using Wiki.Host.Components @using Wiki.Host.Components.Layout diff --git a/src/Wiki.Host/Composition/SyncBackgroundService.cs b/src/Wiki.Host/Composition/SyncBackgroundService.cs new file mode 100644 index 0000000..0dc94c7 --- /dev/null +++ b/src/Wiki.Host/Composition/SyncBackgroundService.cs @@ -0,0 +1,32 @@ +using Wiki.Application.Sync; + +namespace Wiki.Host.Composition; + +/// Syncs at start, on every webhook trigger, and every interval; retries sooner after a failure. +public sealed class SyncBackgroundService(SyncCoordinator coordinator, WikiOptions options, ILogger log) : BackgroundService +{ + private readonly SemaphoreSlim _wake = new(0); + + public void Trigger() => _wake.Release(); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + var summary = await coordinator.RunAsync(force: false, stoppingToken); + var wait = summary is null ? options.Sync.RetrySeconds : options.Sync.IntervalSeconds; + if (summary is null) + log.LogWarning("sync failed; retrying in {Seconds}s", wait); + try + { + await _wake.WaitAsync(TimeSpan.FromSeconds(wait), stoppingToken); + } + catch (OperationCanceledException) + { + break; + } + // drain extra triggers that arrived during the run; one sync covers them + while (_wake.CurrentCount > 0) await _wake.WaitAsync(0, stoppingToken); + } + } +} diff --git a/src/Wiki.Host/Composition/WikiOptions.cs b/src/Wiki.Host/Composition/WikiOptions.cs new file mode 100644 index 0000000..2b39673 --- /dev/null +++ b/src/Wiki.Host/Composition/WikiOptions.cs @@ -0,0 +1,55 @@ +namespace Wiki.Host.Composition; + +/// The `Wiki` configuration section: which adapters exist and which one is active. Secrets are read separately from configuration keys, never bound here. +public sealed class WikiOptions +{ + public NoteStoreOptions NoteStore { get; set; } = new(); + public EmbeddingsOptions Embeddings { get; set; } = new(); + public IndexOptions Index { get; set; } = new(); + public SyncScheduleOptions Sync { get; set; } = new(); +} + +public sealed class NoteStoreOptions +{ + public string Active { get; set; } = "gitea"; + public GiteaStoreOptions Gitea { get; set; } = new(); +} + +public sealed class GiteaStoreOptions +{ + public string BaseUrl { get; set; } = "https://gitea.renner.uno"; + public string Repository { get; set; } = "k.renner/notes"; + public string Branch { get; set; } = "main"; +} + +public sealed class EmbeddingsOptions +{ + public string Active { get; set; } = "bge-m3"; + public Dictionary Models { get; set; } = new(); +} + +public sealed class EmbeddingModelOptions +{ + public string Adapter { get; set; } = "LlamaCpp"; + public string Url { get; set; } = string.Empty; + public string Model { get; set; } = string.Empty; + public int Dimension { get; set; } = 1024; +} + +public sealed class IndexOptions +{ + public string Active { get; set; } = "file"; + public FileIndexStoreOptions File { get; set; } = new(); +} + +public sealed class FileIndexStoreOptions +{ + public string Directory { get; set; } = "/data"; +} + +public sealed class SyncScheduleOptions +{ + /// Reconcile interval; the webhook is the fast path. + public int IntervalSeconds { get; set; } = 1800; + public int RetrySeconds { get; set; } = 60; +} diff --git a/src/Wiki.Host/Program.cs b/src/Wiki.Host/Program.cs index f643367..17fc409 100644 --- a/src/Wiki.Host/Program.cs +++ b/src/Wiki.Host/Program.cs @@ -1,27 +1,105 @@ +using Microsoft.AspNetCore.Authentication; +using MudBlazor.Services; +using Wiki.Adapters.FileIndex; +using Wiki.Adapters.Gitea; +using Wiki.Adapters.LlamaCpp; +using Wiki.Application.Retrieval; +using Wiki.Application.Status; +using Wiki.Application.Sync; +using Wiki.Core.Ports; +using Wiki.Host.Api; +using Wiki.Host.Auth; using Wiki.Host.Components; +using Wiki.Host.Composition; var builder = WebApplication.CreateBuilder(args); +var wiki = builder.Configuration.GetSection("Wiki").Get() ?? new WikiOptions(); +builder.Services.AddSingleton(wiki); +builder.Services.AddSingleton(TimeProvider.System); -// Add services to the container. -builder.Services.AddRazorComponents() - .AddInteractiveServerComponents(); +// ---- adapters, selected by configuration ------------------------------------------------ +builder.Services.AddGiteaAdapters(o => +{ + o.BaseUrl = wiki.NoteStore.Gitea.BaseUrl; + o.Repository = wiki.NoteStore.Gitea.Repository; + o.Branch = wiki.NoteStore.Gitea.Branch; + o.Token = builder.Configuration["Wiki:NoteStore:Gitea:Token"] ?? string.Empty; +}); +builder.Services.AddFileIndexAdapters(o => o.Directory = wiki.Index.File.Directory); +foreach (var (name, model) in wiki.Embeddings.Models) +{ + switch (model.Adapter) + { + case "LlamaCpp": + builder.Services.AddLlamaCppEmbeddings(name, o => + { + o.Url = model.Url; + o.Model = model.Model; + o.ModelId = name; + o.Dimension = model.Dimension; + o.BasicAuth = builder.Configuration[$"Wiki:Embeddings:Models:{name}:BasicAuth"]; + }); + break; + default: + throw new InvalidOperationException($"unknown embeddings adapter '{model.Adapter}' for model '{name}'"); + } +} +builder.Services.AddSingleton>(sp => + wiki.Embeddings.Models.Keys.Select(sp.GetRequiredKeyedService).ToList()); +builder.Services.AddSingleton(sp => sp.GetRequiredKeyedService(wiki.Embeddings.Active)); +builder.Services.AddSingleton>(sp => +[ + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + .. sp.GetRequiredService>(), +]); + +// ---- use cases --------------------------------------------------------------------------- +builder.Services.AddSingleton(new SyncOptions()); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); + +// ---- auth: humans arrive through the Authentik outpost, machines with an API key --------- +builder.Services.AddAuthentication(ProxyHeaderAuthenticationHandler.SchemeName) + .AddScheme(ProxyHeaderAuthenticationHandler.SchemeName, null) + .AddScheme(ApiKeyAuthenticationHandler.SchemeName, null); +builder.Services.AddAuthorization(options => +{ + options.AddPolicy(Policies.Write, p => p.AddAuthenticationSchemes(ApiKeyAuthenticationHandler.SchemeName).RequireAuthenticatedUser()); +}); + +// ---- UI ---------------------------------------------------------------------------------- +builder.Services.AddRazorComponents().AddInteractiveServerComponents(); +builder.Services.AddMudServices(); var app = builder.Build(); -// Configure the HTTP request pipeline. if (!app.Environment.IsDevelopment()) { app.UseExceptionHandler("/Error", createScopeForErrors: true); - // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. - app.UseHsts(); } -app.UseStatusCodePagesWithReExecute("/not-found", createScopeForStatusCodePages: true); -app.UseHttpsRedirection(); - +// The not-found page is for the UI only: an API 401/404 must stay a JSON status, not be +// re-executed as a POST against a Blazor page (which fails antiforgery with an empty 400). +app.UseWhen( + ctx => HttpMethods.IsGet(ctx.Request.Method) && !ctx.Request.Path.StartsWithSegments("/api"), + branch => branch.UseStatusCodePagesWithReExecute("/not-found", createScopeForStatusCodePages: true)); +app.UseAuthentication(); +app.UseAuthorization(); app.UseAntiforgery(); +app.MapHealthEndpoints(); +app.MapIndexEndpoints(); +app.MapApiV1(); + app.MapStaticAssets(); -app.MapRazorComponents() - .AddInteractiveServerRenderMode(); +app.MapRazorComponents().AddInteractiveServerRenderMode(); app.Run(); + +public partial class Program; diff --git a/src/Wiki.Host/Wiki.Host.csproj b/src/Wiki.Host/Wiki.Host.csproj index 90d0b37..2a9ccc1 100644 --- a/src/Wiki.Host/Wiki.Host.csproj +++ b/src/Wiki.Host/Wiki.Host.csproj @@ -9,6 +9,10 @@ + + + + net10.0 enable diff --git a/src/Wiki.Host/appsettings.json b/src/Wiki.Host/appsettings.json index 10f68b8..0a93de3 100644 --- a/src/Wiki.Host/appsettings.json +++ b/src/Wiki.Host/appsettings.json @@ -2,8 +2,27 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Warning", + "System.Net.Http.HttpClient": "Warning" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + "Wiki": { + "NoteStore": { + "Active": "gitea", + "Gitea": { "BaseUrl": "https://gitea.renner.uno", "Repository": "k.renner/notes", "Branch": "main" } + }, + "Embeddings": { + "Active": "bge-m3", + "Models": { + "bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc.cluster.local:8080", "Model": "bge-m3", "Dimension": 1024 } + } + }, + "Index": { "Active": "file", "File": { "Directory": "/data" } }, + "Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 } + }, + "Auth": { + "ReverseProxy": { "HeaderName": "X-Authentik-Username", "TrustedProxies": [] }, + "ApiKeys": {} + } } diff --git a/test/Wiki.Application.Tests/Fakes.cs b/test/Wiki.Application.Tests/Fakes.cs new file mode 100644 index 0000000..2d591a5 --- /dev/null +++ b/test/Wiki.Application.Tests/Fakes.cs @@ -0,0 +1,100 @@ +using System.Security.Cryptography; +using System.Text; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Application.Tests; + +/// In-memory ports for use-case tests. Content hash = SHA-1 of the text, like a git blob. +public sealed class FakeStore : INoteStore +{ + public Dictionary Files { get; } = new(); + public string Head { get; set; } = "c1"; + public int Reads { get; private set; } + public string Name => "fake-store"; + + public Task GetHeadAsync(CancellationToken ct) => Task.FromResult(new StoreHead(Head, null)); + + public Task> ListAsync(string commit, IReadOnlySet kinds, CancellationToken ct) + { + var refs = new List(); + foreach (var (path, text) in Files) + { + var c = DocumentParser.Classify(path); + if (c is null || !kinds.Contains(c.Value.Kind)) continue; + refs.Add(new DocumentRef(c.Value.Kind, c.Value.Slug, path, Convert.ToHexString(SHA1.HashData(Encoding.UTF8.GetBytes(text))))); + } + return Task.FromResult>(refs); + } + + public Task ReadAsync(DocumentRef reference, string commit, CancellationToken ct) + { + Reads++; + return Task.FromResult(Files[reference.Path]); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public sealed class FakeCatalog : IDocumentCatalog +{ + private readonly Dictionary<(DocumentKind, string), Document> _docs = new(); + private CatalogState _state = new(null, null); + public int Flushes { get; private set; } + public string Name => "fake-catalog"; + + public Task GetStateAsync(CancellationToken ct) => Task.FromResult(_state); + public Task SetStateAsync(CatalogState state, CancellationToken ct) { _state = state; return Task.CompletedTask; } + public Task UpsertAsync(IReadOnlyList documents, CancellationToken ct) { foreach (var d in documents) _docs[(d.Kind, d.Slug)] = d; return Task.CompletedTask; } + public Task RemoveAsync(IReadOnlyList references, CancellationToken ct) { foreach (var r in references) _docs.Remove((r.Kind, r.Slug)); return Task.CompletedTask; } + public Task GetAsync(DocumentKind kind, string slug, CancellationToken ct) => Task.FromResult(_docs.GetValueOrDefault((kind, slug))); + public Task> ListAsync(DocumentKind kind, CancellationToken ct) => Task.FromResult>(_docs.Values.Where(d => d.Kind == kind).Select(d => d.Ref).ToList()); + public Task> ListDocumentsAsync(DocumentKind kind, CancellationToken ct) => Task.FromResult>(_docs.Values.Where(d => d.Kind == kind).ToList()); + public Task> BacklinksAsync(string slug, CancellationToken ct) => Task.FromResult>(_docs.Values.Where(d => d.Kind == DocumentKind.Wiki).SelectMany(WikiLinks.Extract).Where(l => l.ToSlug == slug).ToList()); + public Task> OutgoingLinksAsync(string slug, CancellationToken ct) => Task.FromResult>(_docs.TryGetValue((DocumentKind.Wiki, slug), out var d) ? WikiLinks.Extract(d) : []); + public Task> TagCountsAsync(CancellationToken ct) => Task.FromResult>(_docs.Values.Where(d => d.Kind == DocumentKind.Wiki).SelectMany(d => d.Tags).GroupBy(t => t).ToDictionary(g => g.Key, g => g.Count())); + public Task FlushAsync(CancellationToken ct) { Flushes++; return Task.CompletedTask; } + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public sealed class FakeIndex : IVectorIndex +{ + public Dictionary> Models { get; } = new(); + public int Flushes { get; private set; } + public string Name => "fake-index"; + + private Dictionary Of(string model) => Models.TryGetValue(model, out var m) ? m : Models[model] = new(); + + public Task DescribeAsync(string modelId, CancellationToken ct) => Task.FromResult(new IndexState(modelId, Of(modelId).ToDictionary(kv => kv.Key, kv => kv.Value.Hash))); + public Task UpsertAsync(IReadOnlyList embeddings, CancellationToken ct) { foreach (var e in embeddings) Of(e.ModelId)[e.Slug] = (e.ContentHash, e.Vector); return Task.CompletedTask; } + public Task RemoveAsync(string modelId, IReadOnlyList slugs, CancellationToken ct) { foreach (var s in slugs) Of(modelId).Remove(s); return Task.CompletedTask; } + public Task> QueryAsync(string modelId, float[] query, int k, CancellationToken ct) => + Task.FromResult>(Of(modelId).Select(kv => new RankedHit(kv.Key, kv.Value.Vector.Zip(query, (a, b) => a * b).Sum())).OrderByDescending(h => h.Score).Take(k).ToList()); + public Task FlushAsync(CancellationToken ct) { Flushes++; return Task.CompletedTask; } + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +/// Deterministic 4-dimensional "embeddings": a hash of the text spread over the axes, unit length. +public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel +{ + public string ModelId { get; } = modelId; + public int Dimension => 4; + public List> Batches { get; } = []; + public string Name => "fake-embeddings:" + ModelId; + + public Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) + { + Batches.Add(texts); + var vectors = texts.Select(t => + { + var h = SHA256.HashData(Encoding.UTF8.GetBytes(ModelId + t)); + var v = new float[] { h[0] + 1, h[1] + 1, h[2] + 1, h[3] + 1 }; + var n = MathF.Sqrt(v.Sum(x => x * x)); + return v.Select(x => x / n).ToArray(); + }).ToList(); + return Task.FromResult>(vectors); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} diff --git a/test/Wiki.Application.Tests/SyncUseCaseTests.cs b/test/Wiki.Application.Tests/SyncUseCaseTests.cs new file mode 100644 index 0000000..63461e9 --- /dev/null +++ b/test/Wiki.Application.Tests/SyncUseCaseTests.cs @@ -0,0 +1,143 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Wiki.Application.Retrieval; +using Wiki.Application.Sync; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Application.Tests; + +public class SyncUseCaseTests +{ + private static string Note(string title, string body = "Body.", string tags = "[a]") => + $"---\ntitle: {title}\ntype: wiki\ntags: {tags}\nsources: [x]\nupdated: 2026-09-01\n---\n\n# {title}\n\n{body}\n"; + + private readonly FakeStore _store = new(); + private readonly FakeCatalog _catalog = new(); + private readonly FakeIndex _index = new(); + private readonly FakeEmbeddings _model = new("m1"); + + private SyncUseCase Sync(params IEmbeddingModel[] models) => + new(_store, _catalog, _index, models.Length == 0 ? [_model] : models, TimeProvider.System, NullLogger.Instance, new SyncOptions(EmbedChunk: 2)); + + [Fact] + public async Task InitialSyncEmbedsWikiOnlyAndCatalogsEverything() + { + _store.Files["wiki/a.md"] = Note("A"); + _store.Files["wiki/b.md"] = Note("B"); + _store.Files["wiki/c.md"] = Note("C"); + _store.Files["raw/2026-09-23-r.md"] = "---\ntitle: R\n---\n\nraw\n"; + _store.Files["output/2026-09-23-o.md"] = "---\ntitle: O\n---\n\nout\n"; + _store.Files["wiki/_index.md"] = "index"; + + var summary = await Sync().RunAsync(false, CancellationToken.None); + + Assert.False(summary.Skipped); + Assert.Equal(5, summary.Changed); + Assert.Equal(3, summary.Embedded); + Assert.Equal(3, summary.Notes); + Assert.Equal(3, (await _index.DescribeAsync("m1", CancellationToken.None)).Count); + Assert.Single(await _catalog.ListAsync(DocumentKind.Raw, CancellationToken.None)); + Assert.Single(await _catalog.ListAsync(DocumentKind.Output, CancellationToken.None)); + Assert.Equal("c1", (await _catalog.GetStateAsync(CancellationToken.None)).Commit); + Assert.All(_model.Batches, b => Assert.InRange(b.Count, 1, 2)); + Assert.Equal(1, _index.Flushes); + Assert.Equal(1, _catalog.Flushes); + } + + [Fact] + public async Task UnchangedHeadIsSkippedWithoutReads() + { + _store.Files["wiki/a.md"] = Note("A"); + await Sync().RunAsync(false, CancellationToken.None); + var reads = _store.Reads; + + var summary = await Sync().RunAsync(false, CancellationToken.None); + + Assert.True(summary.Skipped); + Assert.Equal(reads, _store.Reads); + Assert.NotNull((await _catalog.GetStateAsync(CancellationToken.None)).SyncedAt); + } + + [Fact] + public async Task ChangedDocumentIsReembeddedAndRemovedOneDropped() + { + _store.Files["wiki/a.md"] = Note("A"); + _store.Files["wiki/b.md"] = Note("B"); + await Sync().RunAsync(false, CancellationToken.None); + var before = (await _index.DescribeAsync("m1", CancellationToken.None)).ContentHashBySlug; + + _store.Head = "c2"; + _store.Files["wiki/a.md"] = Note("A", "Changed body."); + _store.Files.Remove("wiki/b.md"); + var reads = _store.Reads; + + var summary = await Sync().RunAsync(false, CancellationToken.None); + + Assert.Equal(1, summary.Changed); + Assert.Equal(1, summary.Removed); + Assert.Equal(1, _store.Reads - reads); + var after = (await _index.DescribeAsync("m1", CancellationToken.None)).ContentHashBySlug; + Assert.Equal(["a"], after.Keys); + Assert.NotEqual(before["a"], after["a"]); + Assert.Null(await _catalog.GetAsync(DocumentKind.Wiki, "b", CancellationToken.None)); + Assert.Contains("Changed body", (await _catalog.GetAsync(DocumentKind.Wiki, "a", CancellationToken.None))!.Body); + } + + [Fact] + public async Task ForceResyncsAnUnchangedHead() + { + _store.Files["wiki/a.md"] = Note("A"); + await Sync().RunAsync(false, CancellationToken.None); + var summary = await Sync().RunAsync(true, CancellationToken.None); + Assert.False(summary.Skipped); + Assert.Equal(0, summary.Changed); + } + + [Fact] + public async Task ALaterRegisteredModelIsBackfilledFromTheCatalog() + { + _store.Files["wiki/a.md"] = Note("A"); + _store.Files["wiki/b.md"] = Note("B"); + await Sync().RunAsync(false, CancellationToken.None); + + var second = new FakeEmbeddings("m2"); + var summary = await Sync(_model, second).RunAsync(true, CancellationToken.None); + + Assert.Equal(2, summary.Embedded); + Assert.Equal(2, (await _index.DescribeAsync("m2", CancellationToken.None)).Count); + Assert.Equal(2, (await _index.DescribeAsync("m1", CancellationToken.None)).Count); + } + + [Fact] + public async Task RankReturnsTitlesFromTheCatalog() + { + _store.Files["wiki/udm-dns.md"] = Note("UDM intercepts DNS", "The UDM answers DNS."); + _store.Files["wiki/ceph-osd.md"] = Note("Ceph OSD", "OSD fsid."); + await Sync().RunAsync(false, CancellationToken.None); + + var rank = new RankUseCase(_index, _catalog); + var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!; + var rows = await rank.RankAsync(_model, Wiki.Core.Parsing.EmbeddingText.For(udm), 5, CancellationToken.None); + + Assert.Equal(2, rows.Count); + Assert.Equal("udm-dns", rows[0].Slug); + Assert.Equal("UDM intercepts DNS", rows[0].Title); + Assert.InRange(rows[0].Score, 0.99f, 1.01f); + await Assert.ThrowsAsync(() => rank.RankAsync(_model, " ", 5, CancellationToken.None)); + } + + [Fact] + public async Task CoordinatorRecordsFailureAndRecovers() + { + _store.Files["wiki/a.md"] = Note("A"); + var coordinator = new SyncCoordinator(Sync(), TimeProvider.System, NullLogger.Instance); + _store.Head = null!; // GetHeadAsync returns a null commit → the use case fails on Substring + Assert.Null(await coordinator.RunAsync(false, CancellationToken.None)); + Assert.NotNull(coordinator.LastError); + Assert.False(coordinator.Syncing); + + _store.Head = "c1"; + Assert.NotNull(await coordinator.RunAsync(false, CancellationToken.None)); + Assert.Null(coordinator.LastError); + } +} diff --git a/test/Wiki.Application.Tests/Wiki.Application.Tests.csproj b/test/Wiki.Application.Tests/Wiki.Application.Tests.csproj index 61e9574..9d03af8 100644 --- a/test/Wiki.Application.Tests/Wiki.Application.Tests.csproj +++ b/test/Wiki.Application.Tests/Wiki.Application.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/test/Wiki.Contracts/DocumentCatalogContract.cs b/test/Wiki.Contracts/DocumentCatalogContract.cs new file mode 100644 index 0000000..d81bd30 --- /dev/null +++ b/test/Wiki.Contracts/DocumentCatalogContract.cs @@ -0,0 +1,112 @@ +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Contracts; + +/// Every adapter derives from this. +public abstract class DocumentCatalogContract +{ + protected abstract Task CreateCatalogAsync(); + protected abstract Task ReopenAsync(IDocumentCatalog catalog); + + private static Document Doc(string path, string hash = "h") => + DocumentParser.Parse(Ref(path, hash), VaultFixture.Files[path]); + + private static DocumentRef Ref(string path, string hash = "h") + { + var (kind, slug) = DocumentParser.Classify(path)!.Value; + return new DocumentRef(kind, slug, path, hash); + } + + [Fact] + public async Task StartsEmptyWithNoState() + { + var catalog = await CreateCatalogAsync(); + var state = await catalog.GetStateAsync(CancellationToken.None); + Assert.Null(state.Commit); + Assert.Empty(await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None)); + } + + [Fact] + public async Task UpsertGetListByKind() + { + var catalog = await CreateCatalogAsync(); + await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md"), Doc("raw/2026-09-23-pending.md"), Doc("output/2026-09-20-answer.md")], CancellationToken.None); + + var got = await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None); + Assert.NotNull(got); + Assert.Equal("UDM intercepts DNS", got.Title); + Assert.Contains("answers every UDP/53", got.Body); + Assert.Null(await catalog.GetAsync(DocumentKind.Raw, "udm-dns", CancellationToken.None)); + + Assert.Equal(2, (await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None)).Count); + Assert.Single(await catalog.ListAsync(DocumentKind.Raw, CancellationToken.None)); + Assert.Single(await catalog.ListDocumentsAsync(DocumentKind.Output, CancellationToken.None)); + } + + [Fact] + public async Task LinksAndBacklinksFollowWikilinksAndRelated() + { + var catalog = await CreateCatalogAsync(); + await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None); + var outgoing = await catalog.OutgoingLinksAsync("udm-dns", CancellationToken.None); + Assert.Equal([LinkKind.Related, LinkKind.WikiLink], outgoing.Select(l => l.Kind)); + Assert.All(outgoing, l => Assert.Equal("ceph-osd", l.ToSlug)); + var back = await catalog.BacklinksAsync("ceph-osd", CancellationToken.None); + Assert.Equal(2, back.Count); + Assert.All(back, l => Assert.Equal("udm-dns", l.FromSlug)); + Assert.Empty(await catalog.BacklinksAsync("udm-dns", CancellationToken.None)); + } + + [Fact] + public async Task RemoveDropsDocumentAndItsLinks() + { + var catalog = await CreateCatalogAsync(); + await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None); + await catalog.RemoveAsync([Ref("wiki/udm-dns.md")], CancellationToken.None); + Assert.Null(await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None)); + Assert.Empty(await catalog.BacklinksAsync("ceph-osd", CancellationToken.None)); + } + + [Fact] + public async Task UpsertReplacesLinks() + { + var catalog = await CreateCatalogAsync(); + await catalog.UpsertAsync([Doc("wiki/udm-dns.md")], CancellationToken.None); + var changed = Doc("wiki/udm-dns.md", "h2") with { Body = "no links any more", Related = [] }; + await catalog.UpsertAsync([changed], CancellationToken.None); + Assert.Empty(await catalog.OutgoingLinksAsync("udm-dns", CancellationToken.None)); + Assert.Equal("h2", (await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!.ContentHash); + } + + [Fact] + public async Task TagCountsCoverWikiOnly() + { + var catalog = await CreateCatalogAsync(); + await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md"), Doc("output/2026-09-20-answer.md")], CancellationToken.None); + var tags = await catalog.TagCountsAsync(CancellationToken.None); + Assert.Equal(1, tags["dns"]); + Assert.Equal(1, tags["udm"]); + Assert.Equal(1, tags["ceph"]); + Assert.Equal(3, tags.Count); + } + + [Fact] + public async Task StateAndDocumentsSurviveFlushAndReopen() + { + var catalog = await CreateCatalogAsync(); + await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None); + var at = new DateTimeOffset(2026, 9, 24, 10, 0, 0, TimeSpan.Zero); + await catalog.SetStateAsync(new CatalogState(VaultFixture.Commit, at), CancellationToken.None); + await catalog.FlushAsync(CancellationToken.None); + + var reopened = await ReopenAsync(catalog); + var state = await reopened.GetStateAsync(CancellationToken.None); + Assert.Equal(VaultFixture.Commit, state.Commit); + Assert.Equal(at, state.SyncedAt); + Assert.Equal(2, (await reopened.ListAsync(DocumentKind.Wiki, CancellationToken.None)).Count); + Assert.Equal(2, (await reopened.BacklinksAsync("ceph-osd", CancellationToken.None)).Count); + Assert.Equal("UDM intercepts DNS", (await reopened.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!.Title); + } +} diff --git a/test/Wiki.Contracts/EmbeddingModelContract.cs b/test/Wiki.Contracts/EmbeddingModelContract.cs new file mode 100644 index 0000000..ff47987 --- /dev/null +++ b/test/Wiki.Contracts/EmbeddingModelContract.cs @@ -0,0 +1,49 @@ +using Wiki.Core.Ports; + +namespace Wiki.Contracts; + +/// Every adapter derives from this. The adapter under test may talk to a fake server. +public abstract class EmbeddingModelContract +{ + protected abstract Task CreateModelAsync(); + + [Fact] + public async Task VectorsHaveTheDeclaredDimensionAndUnitLength() + { + var model = await CreateModelAsync(); + var vectors = await model.EmbedAsync(["alpha", "beta"], CancellationToken.None); + Assert.Equal(2, vectors.Count); + foreach (var v in vectors) + { + Assert.Equal(model.Dimension, v.Length); + var norm = MathF.Sqrt(v.Sum(x => x * x)); + Assert.InRange(norm, 0.99f, 1.01f); + } + } + + [Fact] + public async Task SameTextSameVectorAndOrderPreserved() + { + var model = await CreateModelAsync(); + var once = await model.EmbedAsync(["one", "two", "three"], CancellationToken.None); + var again = await model.EmbedAsync(["three", "one"], CancellationToken.None); + Assert.Equal(once[2], again[0]); + Assert.Equal(once[0], again[1]); + } + + [Fact] + public async Task EmptyBatchIsEmpty() + { + var model = await CreateModelAsync(); + Assert.Empty(await model.EmbedAsync([], CancellationToken.None)); + } + + [Fact] + public async Task ModelIdIsSet() + { + var model = await CreateModelAsync(); + Assert.False(string.IsNullOrWhiteSpace(model.ModelId)); + var health = await model.CheckAsync(CancellationToken.None); + Assert.True(health.Ok, health.Detail); + } +} diff --git a/test/Wiki.Contracts/FileIndex/FileDocumentCatalogContractTests.cs b/test/Wiki.Contracts/FileIndex/FileDocumentCatalogContractTests.cs new file mode 100644 index 0000000..6367a25 --- /dev/null +++ b/test/Wiki.Contracts/FileIndex/FileDocumentCatalogContractTests.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Options; +using Wiki.Adapters.FileIndex; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.FileIndex; + +public sealed class FileDocumentCatalogContractTests : DocumentCatalogContract, IDisposable +{ + private readonly List _directories = []; + private readonly Dictionary _directoryOf = new(ReferenceEqualityComparer.Instance); + + protected override Task CreateCatalogAsync() + { + var directory = NewDirectory(); + IDocumentCatalog catalog = new Wiki.Adapters.FileIndex.FileDocumentCatalog(Options.Create(new FileIndexOptions { Directory = directory })); + _directoryOf[catalog] = directory; + return Task.FromResult(catalog); + } + + protected override Task ReopenAsync(IDocumentCatalog catalog) + { + var directory = _directoryOf[catalog]; + IDocumentCatalog reopened = new Wiki.Adapters.FileIndex.FileDocumentCatalog(Options.Create(new FileIndexOptions { Directory = directory })); + _directoryOf[reopened] = directory; + return Task.FromResult(reopened); + } + + private string NewDirectory() + { + var directory = Path.Combine(Path.GetTempPath(), "wiki-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + _directories.Add(directory); + return directory; + } + + public void Dispose() + { + foreach (var directory in _directories) + { + try { Directory.Delete(directory, recursive: true); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + } +} diff --git a/test/Wiki.Contracts/FileIndex/FileIndexUnitTests.cs b/test/Wiki.Contracts/FileIndex/FileIndexUnitTests.cs new file mode 100644 index 0000000..59902a3 --- /dev/null +++ b/test/Wiki.Contracts/FileIndex/FileIndexUnitTests.cs @@ -0,0 +1,98 @@ +using Microsoft.Extensions.Options; +using Wiki.Adapters.FileIndex; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.FileIndex; + +/// Behaviour specific to the file-backed adapters that the shared ports contracts don't exercise: atomicity and resilience to corrupt state on disk. +public sealed class FileIndexUnitTests : IDisposable +{ + private readonly List _directories = []; + + private string NewDirectory() + { + var directory = Path.Combine(Path.GetTempPath(), "wiki-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + _directories.Add(directory); + return directory; + } + + private static IOptions OptionsFor(string directory) => Options.Create(new FileIndexOptions { Directory = directory }); + + private static float[] Unit(params float[] v) + { + var n = MathF.Sqrt(v.Sum(x => x * x)); + return v.Select(x => x / n).ToArray(); + } + + [Fact] + public async Task VectorIndexFlushLeavesNoTempFileBehind() + { + var directory = NewDirectory(); + var index = new FileVectorIndex(OptionsFor(directory)); + await index.UpsertAsync([new Embedding("m", "a", "h1", Unit(1, 0, 0))], CancellationToken.None); + await index.FlushAsync(CancellationToken.None); + + var indexDirectory = Path.Combine(directory, "index"); + Assert.Empty(Directory.GetFiles(indexDirectory, "*.tmp")); + Assert.NotEmpty(Directory.GetFiles(indexDirectory, "*.bin")); + } + + [Fact] + public async Task VectorIndexCorruptFileStartsEmptyAndHealthMentionsReset() + { + var directory = NewDirectory(); + var indexDirectory = Path.Combine(directory, "index"); + Directory.CreateDirectory(indexDirectory); + await File.WriteAllBytesAsync(Path.Combine(indexDirectory, "broken.bin"), [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + + var index = new FileVectorIndex(OptionsFor(directory)); + var state = await index.DescribeAsync("test-model", CancellationToken.None); + Assert.Equal(0, state.Count); + + var health = await index.CheckAsync(CancellationToken.None); + Assert.True(health.Ok); + Assert.NotNull(health.Detail); + Assert.Contains("reset", health.Detail, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CatalogFlushLeavesNoTempFileBehind() + { + var directory = NewDirectory(); + var catalog = new FileDocumentCatalog(OptionsFor(directory)); + await catalog.SetStateAsync(new CatalogState("abc", DateTimeOffset.UtcNow), CancellationToken.None); + await catalog.FlushAsync(CancellationToken.None); + + Assert.Empty(Directory.GetFiles(directory, "*.tmp")); + Assert.True(File.Exists(Path.Combine(directory, "catalog.json"))); + } + + [Fact] + public async Task CatalogCorruptFileStartsEmptyAndHealthMentionsReset() + { + var directory = NewDirectory(); + await File.WriteAllTextAsync(Path.Combine(directory, "catalog.json"), "{ not valid json"); + + var catalog = new FileDocumentCatalog(OptionsFor(directory)); + var state = await catalog.GetStateAsync(CancellationToken.None); + Assert.Null(state.Commit); + Assert.Empty(await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None)); + + var health = await catalog.CheckAsync(CancellationToken.None); + Assert.True(health.Ok); + Assert.NotNull(health.Detail); + Assert.Contains("reset", health.Detail, StringComparison.OrdinalIgnoreCase); + } + + public void Dispose() + { + foreach (var directory in _directories) + { + try { Directory.Delete(directory, recursive: true); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + } +} diff --git a/test/Wiki.Contracts/FileIndex/FileVectorIndexContractTests.cs b/test/Wiki.Contracts/FileIndex/FileVectorIndexContractTests.cs new file mode 100644 index 0000000..4438f21 --- /dev/null +++ b/test/Wiki.Contracts/FileIndex/FileVectorIndexContractTests.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Options; +using Wiki.Adapters.FileIndex; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.FileIndex; + +public sealed class FileVectorIndexContractTests : VectorIndexContract, IDisposable +{ + private readonly List _directories = []; + private readonly Dictionary _directoryOf = new(ReferenceEqualityComparer.Instance); + + protected override Task CreateIndexAsync() + { + var directory = NewDirectory(); + IVectorIndex index = new Wiki.Adapters.FileIndex.FileVectorIndex(Options.Create(new FileIndexOptions { Directory = directory })); + _directoryOf[index] = directory; + return Task.FromResult(index); + } + + protected override Task ReopenAsync(IVectorIndex index) + { + var directory = _directoryOf[index]; + IVectorIndex reopened = new Wiki.Adapters.FileIndex.FileVectorIndex(Options.Create(new FileIndexOptions { Directory = directory })); + _directoryOf[reopened] = directory; + return Task.FromResult(reopened); + } + + private string NewDirectory() + { + var directory = Path.Combine(Path.GetTempPath(), "wiki-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + _directories.Add(directory); + return directory; + } + + public void Dispose() + { + foreach (var directory in _directories) + { + try { Directory.Delete(directory, recursive: true); } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + } +} diff --git a/test/Wiki.Contracts/Gitea/FakeGiteaHandler.cs b/test/Wiki.Contracts/Gitea/FakeGiteaHandler.cs new file mode 100644 index 0000000..0437e8d --- /dev/null +++ b/test/Wiki.Contracts/Gitea/FakeGiteaHandler.cs @@ -0,0 +1,202 @@ +using System.Net; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Wiki.Contracts.Gitea; + +/// +/// A minimal Gitea API double, in front of an in-memory copy of . +/// Reproduces the quirks the real Gitea note-store adapter has to tolerate: the tree listing pages +/// with "tree": null past the last page, and may report truncated=true even on a page that turns +/// out to be the last one with data. Blob shas are a SHA-1 hex of the file's content, so they are +/// stable across repeated calls and distinct per distinct content, exactly what the note-store +/// contract requires. +/// +public sealed class FakeGiteaHandler : HttpMessageHandler +{ + private readonly Dictionary _files; + private readonly string _repo; + private readonly string _branch; + private readonly string _commit; + private readonly int _pageSize; + + /// Every file created via POST .../contents/{path}, path -> decoded text, in call order. + public List<(string Path, string Content, string Branch, string Message)> Created { get; } = []; + + public FakeGiteaHandler(string repository = "k.renner/notes", string branch = "main", int pageSize = 3) + { + _repo = repository; + _branch = branch; + _commit = VaultFixture.Commit; + _pageSize = pageSize; + _files = new Dictionary(VaultFixture.Files); + } + + /// Adds a file to the fake as if it already existed at HEAD, e.g. to force the + /// fact sink into its -2 suffix path. + public void SeedFile(string path, string content) => _files[path] = content; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + var path = request.RequestUri!.AbsolutePath; + var prefix = $"/api/v1/repos/{_repo}/"; + if (!path.StartsWith(prefix, StringComparison.Ordinal)) + return Task.FromResult(NotFound()); + + var rest = path[prefix.Length..]; + + if (request.Method == HttpMethod.Get && rest == $"branches/{_branch}") + return Task.FromResult(Json(new JsonObject + { + ["commit"] = new JsonObject + { + ["id"] = _commit, + ["timestamp"] = "2026-09-20T12:00:00Z", + }, + })); + + if (request.Method == HttpMethod.Get && rest.StartsWith($"git/trees/{_commit}", StringComparison.Ordinal)) + return Task.FromResult(HandleTree(request.RequestUri)); + + if (request.Method == HttpMethod.Get && rest.StartsWith("raw/", StringComparison.Ordinal)) + return Task.FromResult(HandleRaw(rest["raw/".Length..])); + + if (request.Method == HttpMethod.Get && rest.StartsWith("contents/", StringComparison.Ordinal)) + return Task.FromResult(HandleContentsExists(rest["contents/".Length..])); + + if (request.Method == HttpMethod.Post && rest.StartsWith("contents/", StringComparison.Ordinal)) + return HandleContentsCreateAsync(rest["contents/".Length..], request, ct); + + return Task.FromResult(NotFound()); + } + + private HttpResponseMessage HandleTree(Uri requestUri) + { + var page = QueryInt(requestUri, "page", 1); + var entries = BuildEntries(); + var start = (page - 1) * _pageSize; + + var body = new JsonObject + { + ["sha"] = _commit, + ["total_count"] = entries.Count, + }; + + if (start >= entries.Count) + { + // Gitea answers "tree": null once you page past the end. + body["tree"] = null; + return Json(body); + } + + var slice = entries.Skip(start).Take(_pageSize).ToList(); + var tree = new JsonArray(); + foreach (var e in slice) + { + tree.Add(new JsonObject + { + ["path"] = e.Path, + ["type"] = e.Type, + ["sha"] = e.Sha, + }); + } + body["tree"] = tree; + // Gitea sets this even when this page happens to be the last one with data; callers must + // not use it to decide whether to stop. + body["truncated"] = true; + return Json(body); + } + + private HttpResponseMessage HandleRaw(string encodedPathAndQuery) + { + var filePath = StripQuery(encodedPathAndQuery); + return _files.TryGetValue(filePath, out var content) + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(content, Encoding.UTF8, "text/plain") } + : NotFound(); + } + + private HttpResponseMessage HandleContentsExists(string encodedPathAndQuery) + { + var filePath = StripQuery(encodedPathAndQuery); + return _files.ContainsKey(filePath) + ? Json(new JsonObject { ["path"] = filePath }) + : NotFound(); + } + + private async Task HandleContentsCreateAsync(string encodedPathAndQuery, HttpRequestMessage request, CancellationToken ct) + { + var filePath = StripQuery(encodedPathAndQuery); + var payload = await request.Content!.ReadFromJsonAsync(ct).ConfigureAwait(false) + ?? throw new InvalidOperationException("empty create-file request body"); + var base64 = payload["content"]!.GetValue(); + var branch = payload["branch"]?.GetValue() ?? _branch; + var message = payload["message"]?.GetValue() ?? ""; + var text = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + + if (_files.ContainsKey(filePath)) + return NotFound(HttpStatusCode.UnprocessableEntity, $"{filePath} already exists"); + + _files[filePath] = text; + Created.Add((filePath, text, branch, message)); + + return Json(new JsonObject + { + ["commit"] = new JsonObject { ["sha"] = Sha1Hex(text) }, + }); + } + + private static string StripQuery(string encodedPathAndQuery) + { + var raw = encodedPathAndQuery.Contains('?') ? encodedPathAndQuery[..encodedPathAndQuery.IndexOf('?')] : encodedPathAndQuery; + return Uri.UnescapeDataString(raw); + } + + private static int QueryInt(Uri uri, string key, int fallback) + { + var query = uri.Query.TrimStart('?'); + foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var kv = pair.Split('=', 2); + if (kv.Length == 2 && kv[0] == key && int.TryParse(Uri.UnescapeDataString(kv[1]), out var value)) + return value; + } + return fallback; + } + + private List<(string Path, string Type, string Sha)> BuildEntries() + { + var dirs = new SortedSet(StringComparer.Ordinal); + foreach (var filePath in _files.Keys) + { + var slash = filePath.LastIndexOf('/'); + while (slash > 0) + { + dirs.Add(filePath[..slash]); + slash = filePath.LastIndexOf('/', slash - 1); + } + } + + var entries = new List<(string Path, string Type, string Sha)>(); + foreach (var dir in dirs) + entries.Add((dir, "tree", Sha1Hex($"dir:{dir}"))); + foreach (var (filePath, content) in _files) + entries.Add((filePath, "blob", Sha1Hex(content))); + + return entries.OrderBy(e => e.Path, StringComparer.Ordinal).ToList(); + } + + private static string Sha1Hex(string content) + { + var bytes = SHA1.HashData(Encoding.UTF8.GetBytes(content)); + return Convert.ToHexStringLower(bytes); + } + + private static HttpResponseMessage Json(JsonNode? body) => + new(HttpStatusCode.OK) { Content = new StringContent(body?.ToJsonString() ?? "null", Encoding.UTF8, "application/json") }; + + private static HttpResponseMessage NotFound(HttpStatusCode status = HttpStatusCode.NotFound, string message = "not found") => + new(status) { Content = new StringContent(JsonSerializer.Serialize(new { message }), Encoding.UTF8, "application/json") }; +} diff --git a/test/Wiki.Contracts/Gitea/GiteaFactSinkTests.cs b/test/Wiki.Contracts/Gitea/GiteaFactSinkTests.cs new file mode 100644 index 0000000..96d2042 --- /dev/null +++ b/test/Wiki.Contracts/Gitea/GiteaFactSinkTests.cs @@ -0,0 +1,121 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Options; +using Wiki.Adapters.Gitea; +using Wiki.Core.Model; + +namespace Wiki.Contracts.Gitea; + +public sealed class GiteaFactSinkTests +{ + private const string Repository = "k.renner/notes"; + private const string Branch = "main"; + + private static readonly RawSource Fact = new( + Title: "UDM intercepts DNS: a follow-up", + Source: "https://example.com/thread#5", + Content: "The UDM answers every UDP/53 query itself, independent of what upstream is configured elsewhere.", + ObservedAt: new DateTimeOffset(2026, 9, 20, 14, 30, 0, TimeSpan.Zero), + Tags: ["dns", "udm"]); + + private static readonly Identity By = new("agent-x", IdentityKind.Machine); + private static readonly DateTimeOffset Now = new(2026, 9, 23, 8, 0, 0, TimeSpan.Zero); + + private const string ExpectedPath = "raw/2026-09-23-udm-intercepts-dns-a-follow-up.md"; + + private static readonly string ExpectedText = + "---\n" + + "title: \"UDM intercepts DNS: a follow-up\"\n" + + "type: raw\n" + + "source: \"https://example.com/thread#5\"\n" + + "captured: 2026-09-23\n" + + "captured_by: agent-x\n" + + "observed: 2026-09-20T14:30:00+00:00\n" + + "tags: [dns, udm]\n" + + "---\n" + + "\n" + + "The UDM answers every UDP/53 query itself, independent of what upstream is configured elsewhere.\n"; + + private static (FakeGiteaHandler Handler, GiteaFactSink Sink) CreateSink() + { + var handler = new FakeGiteaHandler(Repository, Branch, pageSize: 3); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://gitea.test") }; + var options = Options.Create(new GiteaOptions + { + BaseUrl = "https://gitea.test", + Repository = Repository, + Branch = Branch, + Token = "test-token", + }); + return (handler, new GiteaFactSink(httpClient, options)); + } + + [Fact] + public async Task WritesTheExpectedPathAndFrontmatter() + { + var (handler, sink) = CreateSink(); + + var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None); + + Assert.Equal(ExpectedPath, receipt.Path); + Assert.Equal(Now, receipt.StoredAt); + + var created = Assert.Single(handler.Created); + Assert.Equal(ExpectedPath, created.Path); + Assert.Equal(ExpectedText, created.Content); + Assert.Equal(Branch, created.Branch); + } + + [Fact] + public async Task ReturnsTheCommitShaFromTheResponse() + { + var (_, sink) = CreateSink(); + + var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None); + + Assert.Equal(Sha1Hex(ExpectedText), receipt.Commit); + } + + [Fact] + public async Task AppendsANumberedSuffixWhenThePathAlreadyExists() + { + var (handler, sink) = CreateSink(); + handler.SeedFile(ExpectedPath, "already here"); + + var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None); + + Assert.Equal("raw/2026-09-23-udm-intercepts-dns-a-follow-up-2.md", receipt.Path); + } + + [Fact] + public async Task KeepsNumberingWhileCandidatesCollide() + { + var (handler, sink) = CreateSink(); + handler.SeedFile(ExpectedPath, "already here"); + handler.SeedFile("raw/2026-09-23-udm-intercepts-dns-a-follow-up-2.md", "also here"); + + var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None); + + Assert.Equal("raw/2026-09-23-udm-intercepts-dns-a-follow-up-3.md", receipt.Path); + } + + [Theory] + [InlineData("", "source", "content long enough to pass the minimum length check, easily.")] + [InlineData("title", "", "content long enough to pass the minimum length check, easily.")] + [InlineData("title", "source", "too short")] + public async Task RejectsAnInvalidRawSourceBeforeCallingGitea(string title, string source, string content) + { + var (handler, sink) = CreateSink(); + var invalid = new RawSource(title, source, content, Now, []); + + await Assert.ThrowsAsync(() => sink.AppendAsync(invalid, By, Now, CancellationToken.None)); + + Assert.Empty(handler.Created); + } + + private static string Sha1Hex(string content) + { + var bytes = SHA1.HashData(Encoding.UTF8.GetBytes(content)); + return Convert.ToHexStringLower(bytes); + } +} diff --git a/test/Wiki.Contracts/Gitea/GiteaNoteStoreContractTests.cs b/test/Wiki.Contracts/Gitea/GiteaNoteStoreContractTests.cs new file mode 100644 index 0000000..59853af --- /dev/null +++ b/test/Wiki.Contracts/Gitea/GiteaNoteStoreContractTests.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.Options; +using Wiki.Adapters.Gitea; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.Gitea; + +public sealed class GiteaNoteStoreContractTests : NoteStoreContract +{ + private const string Repository = "k.renner/notes"; + private const string Branch = "main"; + + protected override Task CreateStoreAsync() + { + var handler = new FakeGiteaHandler(Repository, Branch, pageSize: 3); + var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://gitea.test") }; + var options = Options.Create(new GiteaOptions + { + BaseUrl = "https://gitea.test", + Repository = Repository, + Branch = Branch, + Token = "test-token", + }); + return Task.FromResult(new GiteaNoteStore(httpClient, options)); + } +} diff --git a/test/Wiki.Contracts/Gitea/TransientRetryHandlerTests.cs b/test/Wiki.Contracts/Gitea/TransientRetryHandlerTests.cs new file mode 100644 index 0000000..a1f88e9 --- /dev/null +++ b/test/Wiki.Contracts/Gitea/TransientRetryHandlerTests.cs @@ -0,0 +1,54 @@ +using System.Net; +using Wiki.Adapters.Gitea; + +namespace Wiki.Contracts.Gitea; + +public class TransientRetryHandlerTests +{ + private sealed class Script(params Func[] steps) : HttpMessageHandler + { + public int Calls { get; private set; } + protected override Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + var step = steps[Math.Min(Calls, steps.Length - 1)]; + Calls++; + return Task.FromResult(step()); + } + } + + private static HttpClient Client(Script script, int attempts = 3) => + new(new TransientRetryHandler(attempts, TimeSpan.FromMilliseconds(1), (_, _) => Task.CompletedTask) { InnerHandler = script }); + + [Fact] + public async Task RetriesAGetAfterAConnectionResetAndA502() + { + var script = new Script( + () => throw new HttpRequestException("reset", new IOException("Connection reset by peer")), + () => new HttpResponseMessage(HttpStatusCode.BadGateway), + () => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") }); + var response = await Client(script).GetAsync("http://gitea/x"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(3, script.Calls); + } + + [Fact] + public async Task GivesUpAfterTheConfiguredAttempts() + { + var script = new Script(() => throw new HttpRequestException("reset")); + await Assert.ThrowsAsync(() => Client(script, 3).GetAsync("http://gitea/x")); + Assert.Equal(3, script.Calls); + } + + [Fact] + public async Task DoesNotRetryAPostOrA404() + { + var post = new Script(() => throw new HttpRequestException("reset")); + await Assert.ThrowsAsync(() => Client(post).PostAsync("http://gitea/x", new StringContent("{}"))); + Assert.Equal(1, post.Calls); + + var notFound = new Script(() => new HttpResponseMessage(HttpStatusCode.NotFound)); + var response = await Client(notFound).GetAsync("http://gitea/x"); + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + Assert.Equal(1, notFound.Calls); + } +} diff --git a/test/Wiki.Contracts/LlamaCpp/FakeLlamaCppHandler.cs b/test/Wiki.Contracts/LlamaCpp/FakeLlamaCppHandler.cs new file mode 100644 index 0000000..223af99 --- /dev/null +++ b/test/Wiki.Contracts/LlamaCpp/FakeLlamaCppHandler.cs @@ -0,0 +1,81 @@ +using System.Net; +using System.Text; +using System.Text.Json; + +namespace Wiki.Contracts.LlamaCpp; + +/// +/// Stands in for a llama-server. Serves GET /health (togglable) and POST /v1/embeddings, +/// producing a deterministic, NOT normalised vector per input text and returning rows in a +/// shuffled order tagged with their true "index" so callers must sort by it. +/// +internal sealed class FakeLlamaCppHandler : HttpMessageHandler +{ + public List BatchSizesSeen { get; } = []; + + public bool HealthEndpointAvailable { get; set; } = true; + + public int Dimension { get; init; } = 8; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var path = request.RequestUri!.AbsolutePath; + + if (request.Method == HttpMethod.Get && path.EndsWith("/health", StringComparison.Ordinal)) + { + return HealthEndpointAvailable + ? new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"status\":\"ok\"}") } + : new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (request.Method == HttpMethod.Post && path.EndsWith("/v1/embeddings", StringComparison.Ordinal)) + { + var body = await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + using var doc = JsonDocument.Parse(body); + var texts = doc.RootElement.GetProperty("input").EnumerateArray().Select(e => e.GetString()!).ToArray(); + BatchSizesSeen.Add(texts.Length); + + var rows = texts + .Select((text, index) => new { index, embedding = FakeVector(text) }) + .Reverse() // shuffled: exercises sorting by "index" in the adapter + .ToArray(); + + var json = JsonSerializer.Serialize(new { data = rows }); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + } + + throw new InvalidOperationException($"Unexpected request: {request.Method} {request.RequestUri}"); + } + + private float[] FakeVector(string text) + { + var rng = new Random(StableHash(text)); + var vector = new float[Dimension]; + for (var i = 0; i < Dimension; i++) + { + // Deliberately not unit length: 1..2 range, never zero, so the adapter's own + // normalisation is what makes the contract's unit-length assertion pass. + vector[i] = 1f + rng.NextSingle(); + } + + return vector; + } + + /// FNV-1a over UTF-8 bytes: stable across runs, unlike string.GetHashCode(). + private static int StableHash(string s) + { + unchecked + { + var hash = 2166136261; + foreach (var b in Encoding.UTF8.GetBytes(s)) + { + hash = (hash ^ b) * 16777619; + } + + return (int)hash; + } + } +} diff --git a/test/Wiki.Contracts/LlamaCpp/LlamaCppEmbeddingContractTests.cs b/test/Wiki.Contracts/LlamaCpp/LlamaCppEmbeddingContractTests.cs new file mode 100644 index 0000000..cd0d4fa --- /dev/null +++ b/test/Wiki.Contracts/LlamaCpp/LlamaCppEmbeddingContractTests.cs @@ -0,0 +1,35 @@ +using Wiki.Adapters.LlamaCpp; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.LlamaCpp; + +/// Runs the shared against the llama.cpp adapter over a fake server. +public sealed class LlamaCppEmbeddingContractTests : EmbeddingModelContract +{ + private readonly FakeLlamaCppHandler _handler = new() { Dimension = 8 }; + + protected override Task CreateModelAsync() + { + var client = new HttpClient(_handler) { BaseAddress = new Uri("http://llama.test/") }; + var options = new LlamaCppEmbeddingOptions + { + Url = "http://llama.test/", + Model = "bge-m3", + Dimension = 8, + BatchSize = 2, + }; + + IEmbeddingModel model = new LlamaCppEmbeddingModel(client, options); + return Task.FromResult(model); + } + + [Fact] + public async Task RequestsAreBatchedByTheConfiguredSize() + { + var model = await CreateModelAsync(); + + await model.EmbedAsync(["a", "b", "c", "d", "e"], CancellationToken.None); + + Assert.Equal([2, 2, 1], _handler.BatchSizesSeen); + } +} diff --git a/test/Wiki.Contracts/LlamaCpp/LlamaCppEmbeddingModelTests.cs b/test/Wiki.Contracts/LlamaCpp/LlamaCppEmbeddingModelTests.cs new file mode 100644 index 0000000..ce03a63 --- /dev/null +++ b/test/Wiki.Contracts/LlamaCpp/LlamaCppEmbeddingModelTests.cs @@ -0,0 +1,56 @@ +using Wiki.Adapters.LlamaCpp; + +namespace Wiki.Contracts.LlamaCpp; + +public sealed class LlamaCppEmbeddingModelTests +{ + private static LlamaCppEmbeddingModel CreateModel(FakeLlamaCppHandler handler, int dimension = 8) + { + var client = new HttpClient(handler) { BaseAddress = new Uri("http://llama.test/") }; + var options = new LlamaCppEmbeddingOptions + { + Url = "http://llama.test/", + Model = "bge-m3", + Dimension = dimension, + }; + + return new LlamaCppEmbeddingModel(client, options); + } + + [Fact] + public async Task DimensionMismatchThrowsNamingActualAndExpected() + { + var handler = new FakeLlamaCppHandler { Dimension = 4 }; + var model = CreateModel(handler, dimension: 8); + + var ex = await Assert.ThrowsAsync( + () => model.EmbedAsync(["hello"], CancellationToken.None)); + + Assert.Contains("4", ex.Message); + Assert.Contains("8", ex.Message); + } + + [Fact] + public async Task CheckAsyncUsesHealthEndpointWithoutEmbedding() + { + var handler = new FakeLlamaCppHandler { HealthEndpointAvailable = true }; + var model = CreateModel(handler); + + var health = await model.CheckAsync(CancellationToken.None); + + Assert.True(health.Ok, health.Detail); + Assert.Empty(handler.BatchSizesSeen); + } + + [Fact] + public async Task CheckAsyncFallsBackToEmbeddingWhenHealthIsMissing() + { + var handler = new FakeLlamaCppHandler { HealthEndpointAvailable = false }; + var model = CreateModel(handler); + + var health = await model.CheckAsync(CancellationToken.None); + + Assert.True(health.Ok, health.Detail); + Assert.Equal([1], handler.BatchSizesSeen); + } +} diff --git a/test/Wiki.Contracts/LlamaCpp/LlamaCppServiceCollectionExtensionsTests.cs b/test/Wiki.Contracts/LlamaCpp/LlamaCppServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000..80c88d5 --- /dev/null +++ b/test/Wiki.Contracts/LlamaCpp/LlamaCppServiceCollectionExtensionsTests.cs @@ -0,0 +1,86 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Wiki.Adapters.LlamaCpp; +using Wiki.Core.Ports; + +namespace Wiki.Contracts.LlamaCpp; + +public sealed class LlamaCppServiceCollectionExtensionsTests +{ + [Fact] + public void RegistersAKeyedEmbeddingModel() + { + var services = new ServiceCollection(); + services.AddLlamaCppEmbeddings("embed", o => + { + o.Url = "http://wiki-embed.ai.svc.cluster.local:8080"; + o.Model = "bge-m3"; + o.Dimension = 1024; + }); + + using var provider = services.BuildServiceProvider(); + var model = provider.GetRequiredKeyedService("embed"); + + Assert.Equal("bge-m3", model.ModelId); + Assert.Equal(1024, model.Dimension); + Assert.Equal("llamacpp-embeddings:bge-m3", model.Name); + } + + [Fact] + public void BasicAuthHeaderIsSentWhenConfigured() + { + var services = new ServiceCollection(); + services.AddLlamaCppEmbeddings("embed", o => + { + o.Url = "http://wiki-embed.example/"; + o.Model = "bge-m3"; + o.Dimension = 8; + o.BasicAuth = "user:pass"; + }); + + using var provider = services.BuildServiceProvider(); + var factory = provider.GetRequiredService(); + var client = factory.CreateClient("llamacpp-embeddings:embed"); + + Assert.NotNull(client.DefaultRequestHeaders.Authorization); + Assert.Equal("Basic", client.DefaultRequestHeaders.Authorization!.Scheme); + Assert.Equal( + Convert.ToBase64String(Encoding.UTF8.GetBytes("user:pass")), + client.DefaultRequestHeaders.Authorization.Parameter); + } + + [Fact] + public void NoAuthorizationHeaderWhenBasicAuthNotConfigured() + { + var services = new ServiceCollection(); + services.AddLlamaCppEmbeddings("embed", o => + { + o.Url = "http://wiki-embed.example/"; + o.Model = "bge-m3"; + o.Dimension = 8; + }); + + using var provider = services.BuildServiceProvider(); + var factory = provider.GetRequiredService(); + var client = factory.CreateClient("llamacpp-embeddings:embed"); + + Assert.Null(client.DefaultRequestHeaders.Authorization); + } + + [Theory] + [InlineData("", "bge-m3", 8)] + [InlineData("http://x/", "", 8)] + [InlineData("http://x/", "bge-m3", 0)] + public void InvalidOptionsAreRejectedEagerly(string url, string model, int dimension) + { + var services = new ServiceCollection(); + + Assert.Throws(() => + services.AddLlamaCppEmbeddings("embed", o => + { + o.Url = url; + o.Model = model; + o.Dimension = dimension; + })); + } +} diff --git a/test/Wiki.Contracts/NoteStoreContract.cs b/test/Wiki.Contracts/NoteStoreContract.cs new file mode 100644 index 0000000..86fc37d --- /dev/null +++ b/test/Wiki.Contracts/NoteStoreContract.cs @@ -0,0 +1,68 @@ +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Contracts; + +/// Every adapter derives a test class from this and supplies a store over . +public abstract class NoteStoreContract +{ + /// A store whose current head exposes exactly . + protected abstract Task CreateStoreAsync(); + + [Fact] + public async Task HeadHasACommit() + { + var store = await CreateStoreAsync(); + var head = await store.GetHeadAsync(CancellationToken.None); + Assert.False(string.IsNullOrWhiteSpace(head.Commit)); + } + + [Fact] + public async Task ListsOnlyVaultDocumentsOfRequestedKinds() + { + var store = await CreateStoreAsync(); + var head = await store.GetHeadAsync(CancellationToken.None); + var all = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None); + Assert.Equal( + VaultFixture.Expected.OrderBy(e => e.Path), + all.Select(r => (r.Kind, r.Slug, r.Path)).OrderBy(e => e.Path)); + + var wiki = await store.ListAsync(head.Commit, new HashSet { DocumentKind.Wiki }, CancellationToken.None); + Assert.All(wiki, r => Assert.Equal(DocumentKind.Wiki, r.Kind)); + Assert.Equal(2, wiki.Count); + } + + [Fact] + public async Task ContentHashIsStableAndDistinct() + { + var store = await CreateStoreAsync(); + var head = await store.GetHeadAsync(CancellationToken.None); + var a = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None); + var b = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None); + Assert.Equal(a.OrderBy(r => r.Path).Select(r => r.ContentHash), b.OrderBy(r => r.Path).Select(r => r.ContentHash)); + Assert.All(a, r => Assert.False(string.IsNullOrWhiteSpace(r.ContentHash))); + Assert.Equal(a.Count, a.Select(r => r.ContentHash).Distinct().Count()); + } + + [Fact] + public async Task ReadsExactText() + { + var store = await CreateStoreAsync(); + var head = await store.GetHeadAsync(CancellationToken.None); + var refs = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None); + foreach (var r in refs) + { + var text = await store.ReadAsync(r, head.Commit, CancellationToken.None); + Assert.Equal(VaultFixture.Files[r.Path], text); + } + } + + [Fact] + public async Task HealthReportsOk() + { + var store = await CreateStoreAsync(); + var health = await store.CheckAsync(CancellationToken.None); + Assert.True(health.Ok, health.Detail); + Assert.Equal(store.Name, health.Adapter); + } +} diff --git a/test/Wiki.Contracts/VaultFixture.cs b/test/Wiki.Contracts/VaultFixture.cs new file mode 100644 index 0000000..acec096 --- /dev/null +++ b/test/Wiki.Contracts/VaultFixture.cs @@ -0,0 +1,38 @@ +using Wiki.Core.Model; + +namespace Wiki.Contracts; + +/// +/// A tiny vault every note-store adapter has to expose identically. Adapters under test are +/// pointed at a fake or a temp copy holding exactly these files; the contract checks that the +/// listing and reads come back as written. +/// +public static class VaultFixture +{ + public const string Commit = "0123456789abcdef0123456789abcdef01234567"; + + public static readonly IReadOnlyDictionary Files = new Dictionary + { + ["wiki/_index.md"] = "---\ntitle: index\ntype: index\n---\n", + ["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns, udm]\nsources: [raw/_processed/2026-09-01-dns.md]\nrelated: [ceph-osd]\nupdated: 2026-09-16\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every UDP/53 query itself. See [[ceph-osd]].\n", + ["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\ntags: [ceph]\nsources: [raw/_processed/2026-09-01-dns.md]\nupdated: 2026-09-10\n---\n\n# Ceph OSD\n\nOSD fsid mismatch after rebuild.\n", + ["raw/2026-09-23-pending.md"] = "---\ntitle: Pending source\ntype: raw\nsource: test\ncaptured: 2026-09-23\n---\n\nVerbatim capture.\n", + ["raw/_processed/2026-09-01-dns.md"] = "---\ntitle: Processed source\ntype: raw\nsource: test\ncaptured: 2026-09-01\n---\n\nVerbatim capture, processed.\n", + ["output/2026-09-20-answer.md"] = "---\ntitle: An answer\ntype: output\nquestion: Does the UDM intercept DNS?\nsources: [udm-dns]\n---\n\nYes.\n", + ["bin/wiki_rank.py"] = "#!/usr/bin/env python3\n", + ["Untitled.md"] = "not a vault document\n", + }; + + /// The refs a listing over all kinds must return (order not significant). + public static readonly IReadOnlyList<(DocumentKind Kind, string Slug, string Path)> Expected = + [ + (DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md"), + (DocumentKind.Wiki, "ceph-osd", "wiki/ceph-osd.md"), + (DocumentKind.Raw, "2026-09-23-pending", "raw/2026-09-23-pending.md"), + (DocumentKind.RawProcessed, "2026-09-01-dns", "raw/_processed/2026-09-01-dns.md"), + (DocumentKind.Output, "2026-09-20-answer", "output/2026-09-20-answer.md"), + ]; + + public static readonly IReadOnlySet AllKinds = + new HashSet { DocumentKind.Wiki, DocumentKind.Raw, DocumentKind.RawProcessed, DocumentKind.Output }; +} diff --git a/test/Wiki.Contracts/VectorIndexContract.cs b/test/Wiki.Contracts/VectorIndexContract.cs new file mode 100644 index 0000000..b9cd218 --- /dev/null +++ b/test/Wiki.Contracts/VectorIndexContract.cs @@ -0,0 +1,105 @@ +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Contracts; + +/// Every adapter derives from this. Vectors are unit length; scores are cosine. +public abstract class VectorIndexContract +{ + /// A fresh, empty index. Called again after semantics are needed. + protected abstract Task CreateIndexAsync(); + + /// The same index reopened from persisted state, or the same instance for adapters that keep no files. + protected abstract Task ReopenAsync(IVectorIndex index); + + private const string Model = "test-model"; + + private static float[] Unit(params float[] v) + { + var n = MathF.Sqrt(v.Sum(x => x * x)); + return v.Select(x => x / n).ToArray(); + } + + [Fact] + public async Task EmptyIndexDescribesEmptyAndQueriesNothing() + { + var index = await CreateIndexAsync(); + var state = await index.DescribeAsync(Model, CancellationToken.None); + Assert.Equal(0, state.Count); + Assert.Empty(await index.QueryAsync(Model, Unit(1, 0, 0), 5, CancellationToken.None)); + } + + [Fact] + public async Task QueryRanksByCosineAndHonoursK() + { + var index = await CreateIndexAsync(); + await index.UpsertAsync( + [ + new Embedding(Model, "a", "h1", Unit(1, 0, 0)), + new Embedding(Model, "b", "h2", Unit(0.9f, 0.1f, 0)), + new Embedding(Model, "c", "h3", Unit(0, 1, 0)), + new Embedding(Model, "d", "h4", Unit(0, 0, 1)), + ], CancellationToken.None); + + var hits = await index.QueryAsync(Model, Unit(1, 0, 0), 2, CancellationToken.None); + Assert.Equal(["a", "b"], hits.Select(h => h.Slug)); + Assert.True(hits[0].Score > hits[1].Score); + Assert.InRange(hits[0].Score, 0.999f, 1.001f); + } + + [Fact] + public async Task UpsertReplacesAndDescribeTracksHashes() + { + var index = await CreateIndexAsync(); + await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0))], CancellationToken.None); + await index.UpsertAsync([new Embedding(Model, "a", "h2", Unit(0, 1, 0))], CancellationToken.None); + var state = await index.DescribeAsync(Model, CancellationToken.None); + Assert.Equal(1, state.Count); + Assert.Equal("h2", state.ContentHashBySlug["a"]); + var hits = await index.QueryAsync(Model, Unit(0, 1, 0), 1, CancellationToken.None); + Assert.Equal("a", hits[0].Slug); + Assert.InRange(hits[0].Score, 0.999f, 1.001f); + } + + [Fact] + public async Task RemoveDropsSlugs() + { + var index = await CreateIndexAsync(); + await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0)), new Embedding(Model, "b", "h2", Unit(0, 1, 0))], CancellationToken.None); + await index.RemoveAsync(Model, ["a", "missing"], CancellationToken.None); + var state = await index.DescribeAsync(Model, CancellationToken.None); + Assert.Equal(["b"], state.ContentHashBySlug.Keys); + } + + [Fact] + public async Task ModelsAreIsolatedNamespaces() + { + var index = await CreateIndexAsync(); + await index.UpsertAsync([new Embedding("m1", "a", "h1", Unit(1, 0, 0))], CancellationToken.None); + await index.UpsertAsync([new Embedding("m2", "b", "h2", Unit(1, 0, 0))], CancellationToken.None); + Assert.Equal(["a"], (await index.QueryAsync("m1", Unit(1, 0, 0), 10, CancellationToken.None)).Select(h => h.Slug)); + Assert.Equal(["b"], (await index.QueryAsync("m2", Unit(1, 0, 0), 10, CancellationToken.None)).Select(h => h.Slug)); + Assert.Equal(1, (await index.DescribeAsync("m1", CancellationToken.None)).Count); + } + + [Fact] + public async Task SurvivesFlushAndReopen() + { + var index = await CreateIndexAsync(); + await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0)), new Embedding(Model, "b", "h2", Unit(0, 1, 0))], CancellationToken.None); + await index.FlushAsync(CancellationToken.None); + var reopened = await ReopenAsync(index); + var state = await reopened.DescribeAsync(Model, CancellationToken.None); + Assert.Equal(2, state.Count); + var hits = await reopened.QueryAsync(Model, Unit(0, 1, 0), 1, CancellationToken.None); + Assert.Equal("b", hits[0].Slug); + } + + [Fact] + public async Task HealthReportsOk() + { + var index = await CreateIndexAsync(); + var health = await index.CheckAsync(CancellationToken.None); + Assert.True(health.Ok, health.Detail); + } +} diff --git a/test/Wiki.Core.Tests/EmbeddingTextTests.cs b/test/Wiki.Core.Tests/EmbeddingTextTests.cs new file mode 100644 index 0000000..b8c9781 --- /dev/null +++ b/test/Wiki.Core.Tests/EmbeddingTextTests.cs @@ -0,0 +1,29 @@ +using Wiki.Core.Model; +using Wiki.Core.Parsing; + +namespace Wiki.Core.Tests; + +public class EmbeddingTextTests +{ + [Fact] + public void ComposesSlugTitleAliasesTagsAndFirstParagraph() + { + var text = "---\ntitle: UDM intercepts DNS\naliases: [udm dns, split horizon]\ntags: [dns, udm]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers\nevery query.\n\nSecond paragraph.\n"; + var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md", "h"), text); + Assert.Equal("udm dns UDM intercepts DNS udm dns split horizon dns udm The UDM answers every query.", EmbeddingText.For(doc)); + } + + [Fact] + public void FirstParagraphNeedsAnH1AndSkipsHeadings() + { + Assert.Equal(string.Empty, EmbeddingText.FirstParagraph("No heading\n\nText.")); + Assert.Equal("Text. More.", EmbeddingText.FirstParagraph("# T\n\n## Sub\n\nText.\nMore.\n\nNext.")); + } + + [Fact] + public void CapsLength() + { + var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "x", "wiki/x.md", "h"), "# x\n\n" + new string('a', 5000)); + Assert.Equal(1500, EmbeddingText.For(doc).Length); + } +} diff --git a/test/Wiki.Core.Tests/ParsingTests.cs b/test/Wiki.Core.Tests/ParsingTests.cs new file mode 100644 index 0000000..f88fa9a --- /dev/null +++ b/test/Wiki.Core.Tests/ParsingTests.cs @@ -0,0 +1,145 @@ +using Wiki.Core.Model; +using Wiki.Core.Parsing; + +namespace Wiki.Core.Tests; + +public class FrontmatterTests +{ + private const string Note = """ + --- + title: UDM intercepts DNS + type: wiki + tags: [dns, udm, "networking, lan"] + aliases: + - udm dns + - 'split horizon' + sources: [raw/_processed/a.md, "live check 2026-09-10: dig @1.1.1.1, dig @10.2.40.99"] + related: [ceph-osd, k8s-gateway-in-cluster-dns] + updated: 2026-09-16 + empty: + --- + + # UDM intercepts DNS + + Body line one. + """; + + [Fact] + public void ParsesScalarsFlowListsBlockListsAndQuotes() + { + var (fields, body, start) = Frontmatter.Parse(Note); + Assert.Equal("UDM intercepts DNS", fields["title"].Scalar); + Assert.Equal(["dns", "udm", "networking, lan"], fields["tags"].List!); + Assert.Equal(["udm dns", "split horizon"], fields["aliases"].List!); + Assert.Equal(["raw/_processed/a.md", "live check 2026-09-10: dig @1.1.1.1, dig @10.2.40.99"], fields["sources"].List!); + Assert.Equal(string.Empty, fields["empty"].Scalar); + Assert.Equal(13, start); + Assert.StartsWith("\n# UDM intercepts DNS", body); + } + + [Fact] + public void NoFrontmatterIsWholeBody() + { + var (fields, body, start) = Frontmatter.Parse("# Just a body\n"); + Assert.Empty(fields); + Assert.Equal("# Just a body\n", body); + Assert.Equal(1, start); + } + + [Fact] + public void UnterminatedFrontmatterIsWholeBody() + { + var (fields, body, _) = Frontmatter.Parse("---\ntitle: x\nno end"); + Assert.Empty(fields); + Assert.Equal("---\ntitle: x\nno end", body); + } +} + +public class DocumentParserTests +{ + [Fact] + public void LiftsVaultFields() + { + var reference = new DocumentRef(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md", "abc"); + var doc = DocumentParser.Parse(reference, FrontmatterTests_Note.Text); + Assert.Equal("UDM intercepts DNS", doc.Title); + Assert.Equal("wiki", doc.Type); + Assert.Equal(new DateOnly(2026, 9, 16), doc.Updated); + Assert.Equal(["ceph-osd", "k8s-gateway-in-cluster-dns"], doc.Related); + Assert.Equal(2, doc.Sources.Count); + Assert.Equal(reference, doc.Ref); + } + + [Fact] + public void TitleFallsBackToSlug() + { + var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Raw, "x", "raw/x.md", "h"), "no frontmatter"); + Assert.Equal("x", doc.Title); + } + + [Theory] + [InlineData("wiki/a-b.md", DocumentKind.Wiki, "a-b")] + [InlineData("raw/2026-09-23-x.md", DocumentKind.Raw, "2026-09-23-x")] + [InlineData("raw/_processed/x.md", DocumentKind.RawProcessed, "x")] + [InlineData("output/2026-09-23-y.md", DocumentKind.Output, "2026-09-23-y")] + public void ClassifiesVaultPaths(string path, DocumentKind kind, string slug) + { + Assert.Equal((kind, slug), DocumentParser.Classify(path)); + Assert.Equal(path, DocumentParser.PathFor(kind, slug)); + } + + [Theory] + [InlineData("wiki/_index.md")] + [InlineData("bin/wiki_rank.py")] + [InlineData("Untitled.md")] + [InlineData("_archive/old.md")] + [InlineData("wiki/sub/x.md")] + public void IgnoresNonDocuments(string path) => Assert.Null(DocumentParser.Classify(path)); +} + +public class WikiLinksTests +{ + [Fact] + public void ExtractsBodyLinksOutsideCodeAndRelated() + { + var text = "---\ntitle: T\nrelated: [rel-one, not a slug]\n---\n\nSee [[target-a]] and [[target-b|alias]] and [[target-c#heading]].\n`[[in-code]]`\n```\n[[fenced]]\n```\n[[ -f \"$X\" ]] and [[396,{{value|int}}]]\n"; + var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "src", "wiki/src.md", "h"), text); + var links = WikiLinks.Extract(doc); + Assert.Equal( + [("rel-one", LinkKind.Related, (int?)null), ("target-a", LinkKind.WikiLink, 6), ("target-b", LinkKind.WikiLink, 6), ("target-c", LinkKind.WikiLink, 6)], + links.Select(l => (l.ToSlug, l.Kind, l.Line)).ToArray()); + } +} + +public class ParagraphChunkerTests +{ + [Fact] + public void SplitsOnBlankLinesHeadingsAndFences() + { + var text = "---\ntitle: UDM\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n\n## Details\n\nEven dig.\nSecond line.\n\n```\ndig @10.2.40.99\n```\n"; + var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "udm", "wiki/udm.md", "h"), text); + var ps = new ParagraphChunker().Split(doc); + Assert.Equal(3, ps.Count); + Assert.Equal("udm#p1", ps[0].Id); + Assert.Equal("UDM intercepts DNS", ps[0].Heading); + Assert.Equal("Details", ps[1].Heading); + Assert.Contains("Second line", ps[1].Text); + Assert.StartsWith("```", ps[2].Text); + } +} + +internal static class FrontmatterTests_Note +{ + public const string Text = """ + --- + title: UDM intercepts DNS + type: wiki + tags: [dns, udm] + sources: [raw/_processed/a.md, "live check: a, b"] + related: [ceph-osd, k8s-gateway-in-cluster-dns] + updated: 2026-09-16 + --- + + # UDM intercepts DNS + """; +} diff --git a/test/Wiki.Host.Tests/EndpointTests.cs b/test/Wiki.Host.Tests/EndpointTests.cs new file mode 100644 index 0000000..1a74703 --- /dev/null +++ b/test/Wiki.Host.Tests/EndpointTests.cs @@ -0,0 +1,145 @@ +using System.Net; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Wiki.Core.Model; +using Wiki.Core.Ports; + +namespace Wiki.Host.Tests; + +/// +/// The host with every adapter replaced by a fake: no network, no files. Exercises the wiring, +/// the endpoint contracts and the auth handlers. +/// +public sealed class WikiHostFactory : WebApplicationFactory +{ + public const string WebhookSecret = "hook-secret"; + public const string ApiKey = "machine-key"; + public FakeStore Store { get; } = new(); + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.UseSetting("Wiki:Webhook:Secret", WebhookSecret); + builder.UseSetting("Auth:ApiKeys:tester", ApiKey); + builder.UseSetting("Auth:ReverseProxy:TrustedProxies:0", "127.0.0.1"); + builder.UseSetting("Wiki:Index:File:Directory", Path.Combine(Path.GetTempPath(), "wiki-host-tests", Guid.NewGuid().ToString("N"))); + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.AddSingleton(Store); + services.RemoveAll(); + services.AddSingleton(new FakeFactSink()); + // The embedding model registered under the active name is replaced by a deterministic fake. + services.RemoveAll(typeof(IEmbeddingModel)); + var fake = new FakeEmbeddings("bge-m3"); + services.AddKeyedSingleton("bge-m3", fake); + services.AddSingleton(fake); + services.AddSingleton>([fake]); + }); + } +} + +public class EndpointTests(WikiHostFactory factory) : IClassFixture +{ + private HttpClient Client => factory.CreateClient(); + + [Fact] + public async Task HealthIsAlwaysOk() + { + var r = await Client.GetAsync("/health"); + Assert.Equal(HttpStatusCode.OK, r.StatusCode); + } + + [Fact] + public async Task ReadyIs503UntilSyncedAndOkAfter() + { + var first = await Client.GetAsync("/ready"); + // The background sync may or may not have completed; force one and check the steady state. + factory.Store.Files["wiki/a.md"] = "---\ntitle: A\ntype: wiki\ntags: [x]\n---\n\n# A\n\nBody.\n"; + var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true"); + req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey); + var sync = await Client.SendAsync(req); + Assert.Equal(HttpStatusCode.OK, sync.StatusCode); + var ready = await Client.GetAsync("/ready"); + Assert.Equal(HttpStatusCode.OK, ready.StatusCode); + Assert.True(first.StatusCode is HttpStatusCode.OK or HttpStatusCode.ServiceUnavailable); + } + + [Fact] + public async Task SyncRequiresAnApiKey() + { + var r = await Client.PostAsync("/api/v1/sync", null); + Assert.True(r.StatusCode == HttpStatusCode.Unauthorized, $"{(int)r.StatusCode}: {await r.Content.ReadAsStringAsync()}"); + var bad = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync"); + bad.Headers.Add("X-Api-Key", "wrong"); + Assert.Equal(HttpStatusCode.Unauthorized, (await Client.SendAsync(bad)).StatusCode); + } + + [Fact] + public async Task RankRejectsEmptyQueryAndUnknownModel() + { + var empty = await Client.PostAsJsonAsync("/rank", new { query = " " }); + Assert.Equal(HttpStatusCode.BadRequest, empty.StatusCode); + var unknown = await Client.PostAsJsonAsync("/rank", new { query = "x", model = "nope" }); + Assert.Equal(HttpStatusCode.BadRequest, unknown.StatusCode); + } + + [Fact] + public async Task RankReturnsRowsInThePythonShape() + { + factory.Store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers.\n"; + var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true"); + req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey); + await Client.SendAsync(req); + + var r = await Client.PostAsJsonAsync("/rank", new { query = "udm dns", k = 3 }); + Assert.Equal(HttpStatusCode.OK, r.StatusCode); + var body = await r.Content.ReadFromJsonAsync(); + Assert.NotNull(body); + Assert.Contains(body.Rows, row => row.Slug == "udm-dns" && row.Title == "UDM intercepts DNS"); + Assert.Equal("bge-m3", body.Model); + } + + [Fact] + public async Task WebhookVerifiesTheGiteaSignature() + { + var payload = Encoding.UTF8.GetBytes("{\"ref\":\"refs/heads/main\"}"); + var good = Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(WikiHostFactory.WebhookSecret), payload)); + + var ok = new HttpRequestMessage(HttpMethod.Post, "/webhook") { Content = new ByteArrayContent(payload) }; + ok.Headers.Add("X-Gitea-Signature", good); + Assert.Equal(HttpStatusCode.OK, (await Client.SendAsync(ok)).StatusCode); + + var bad = new HttpRequestMessage(HttpMethod.Post, "/webhook") { Content = new ByteArrayContent(payload) }; + bad.Headers.Add("X-Gitea-Signature", "deadbeef"); + Assert.Equal(HttpStatusCode.Forbidden, (await Client.SendAsync(bad)).StatusCode); + + var other = Encoding.UTF8.GetBytes("{\"ref\":\"refs/heads/feature\"}"); + var otherSig = Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(WikiHostFactory.WebhookSecret), other)); + var skip = new HttpRequestMessage(HttpMethod.Post, "/webhook") { Content = new ByteArrayContent(other) }; + skip.Headers.Add("X-Gitea-Signature", otherSig); + var skipped = await (await Client.SendAsync(skip)).Content.ReadFromJsonAsync(); + Assert.False(skipped!.Queued); + } + + [Fact] + public async Task StatusListsAdaptersAndIndexes() + { + var status = await Client.GetFromJsonAsync("/api/v1/status"); + Assert.NotNull(status); + Assert.Contains(status.Indexes, i => i.ModelId == "bge-m3" && i.Active); + Assert.NotEmpty(status.Adapters); + } + + private sealed record RankRow(string Slug, string Title, float Score); + private sealed record RankResponse(List Rows, string Model); + private sealed record WebhookResponse(bool Queued); + private sealed record IndexRow(string ModelId, int Notes, bool Active); + private sealed record AdapterRow(string Adapter, bool Ok); + private sealed record StatusResponse(bool Ready, List Indexes, List Adapters); +} diff --git a/test/Wiki.Host.Tests/Fakes.cs b/test/Wiki.Host.Tests/Fakes.cs new file mode 100644 index 0000000..764cba8 --- /dev/null +++ b/test/Wiki.Host.Tests/Fakes.cs @@ -0,0 +1,66 @@ +using System.Security.Cryptography; +using System.Text; +using Wiki.Core.Model; +using Wiki.Core.Parsing; +using Wiki.Core.Ports; + +namespace Wiki.Host.Tests; + +public sealed class FakeStore : INoteStore +{ + public Dictionary Files { get; } = new(); + public string Name => "fake-store"; + private int _version; + + public Task GetHeadAsync(CancellationToken ct) + { + // The head changes whenever the files do, like a repository would. + var digest = SHA1.HashData(Encoding.UTF8.GetBytes(string.Join("\n", Files.OrderBy(f => f.Key).Select(f => f.Key + f.Value)))); + _version++; + return Task.FromResult(new StoreHead(Convert.ToHexStringLower(digest), null)); + } + + public Task> ListAsync(string commit, IReadOnlySet kinds, CancellationToken ct) + { + var refs = new List(); + foreach (var (path, text) in Files) + { + var c = DocumentParser.Classify(path); + if (c is null || !kinds.Contains(c.Value.Kind)) continue; + refs.Add(new DocumentRef(c.Value.Kind, c.Value.Slug, path, Convert.ToHexString(SHA1.HashData(Encoding.UTF8.GetBytes(text))))); + } + return Task.FromResult>(refs); + } + + public Task ReadAsync(DocumentRef reference, string commit, CancellationToken ct) => Task.FromResult(Files[reference.Path]); + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public sealed class FakeFactSink : IFactSink +{ + public string Name => "fake-facts"; + public Task AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct) => + Task.FromResult(new FactReceipt("raw/fake.md", "c", now)); + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} + +public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel +{ + public string ModelId { get; } = modelId; + public int Dimension => 4; + public string Name => "fake-embeddings:" + ModelId; + + public Task> EmbedAsync(IReadOnlyList texts, CancellationToken ct) + { + var vectors = texts.Select(t => + { + var h = SHA256.HashData(Encoding.UTF8.GetBytes(t)); + var v = new float[] { h[0] + 1, h[1] + 1, h[2] + 1, h[3] + 1 }; + var n = MathF.Sqrt(v.Sum(x => x * x)); + return v.Select(x => x / n).ToArray(); + }).ToList(); + return Task.FromResult>(vectors); + } + + public Task CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch)); +} diff --git a/test/Wiki.Host.Tests/Wiki.Host.Tests.csproj b/test/Wiki.Host.Tests/Wiki.Host.Tests.csproj index 6dab6d6..dd86386 100644 --- a/test/Wiki.Host.Tests/Wiki.Host.Tests.csproj +++ b/test/Wiki.Host.Tests/Wiki.Host.Tests.csproj @@ -9,6 +9,7 @@ +