Phase 0+1: solution skeleton, ports, adapters, host with parity to the Python wiki-index #1
@@ -0,0 +1,31 @@
|
||||
when:
|
||||
- event: push
|
||||
branch: main
|
||||
|
||||
depends_on:
|
||||
- test
|
||||
|
||||
steps:
|
||||
- name: build-image
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||
environment:
|
||||
GITEA_TOKEN:
|
||||
from_secret: registry_token
|
||||
commands:
|
||||
- mkdir -p /kaniko/.docker
|
||||
- REGISTRY=$${CI_FORGE_URL#*://}
|
||||
- IMAGE="$$REGISTRY/$$CI_REPO"
|
||||
# The repo owner is an organisation; the registry login is the bot in its Bots team.
|
||||
- AUTH=$(printf "%s" "claude-bot:$GITEA_TOKEN" | base64 -w0)
|
||||
- printf '{"auths":{"%s":{"auth":"%s"}}}' "$$REGISTRY" "$AUTH" > /kaniko/.docker/config.json
|
||||
- BUILD_TAG="build.$(date +%s)"
|
||||
- >-
|
||||
/kaniko/executor
|
||||
--context=.
|
||||
--dockerfile=Dockerfile
|
||||
--destination=$$IMAGE:${CI_COMMIT_SHA}
|
||||
--destination=$$IMAGE:${CI_COMMIT_SHA:0:7}
|
||||
--destination=$$IMAGE:$${BUILD_TAG}
|
||||
--destination=$$IMAGE:main
|
||||
--snapshot-mode=redo
|
||||
--use-new-run
|
||||
@@ -0,0 +1,10 @@
|
||||
when:
|
||||
- event: [pull_request, push, manual]
|
||||
|
||||
steps:
|
||||
- name: test
|
||||
image: mcr.microsoft.com/dotnet/sdk:10.0
|
||||
commands:
|
||||
- dotnet restore Wiki.slnx
|
||||
- dotnet build Wiki.slnx --no-restore -c Release
|
||||
- dotnet test Wiki.slnx --no-build -c Release --logger "console;verbosity=minimal"
|
||||
@@ -0,0 +1,10 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<InvariantGlobalization>true</InvariantGlobalization>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||
WORKDIR /src
|
||||
|
||||
COPY Wiki.slnx Directory.Build.props ./
|
||||
COPY src/Wiki.Core/Wiki.Core.csproj src/Wiki.Core/
|
||||
COPY src/Wiki.Application/Wiki.Application.csproj src/Wiki.Application/
|
||||
COPY src/Wiki.Adapters.Gitea/Wiki.Adapters.Gitea.csproj src/Wiki.Adapters.Gitea/
|
||||
COPY src/Wiki.Adapters.LlamaCpp/Wiki.Adapters.LlamaCpp.csproj src/Wiki.Adapters.LlamaCpp/
|
||||
COPY src/Wiki.Adapters.TypeSafe/Wiki.Adapters.TypeSafe.csproj src/Wiki.Adapters.TypeSafe/
|
||||
COPY src/Wiki.Adapters.FileIndex/Wiki.Adapters.FileIndex.csproj src/Wiki.Adapters.FileIndex/
|
||||
COPY src/Wiki.Adapters.Anthropic/Wiki.Adapters.Anthropic.csproj src/Wiki.Adapters.Anthropic/
|
||||
COPY src/Wiki.Host/Wiki.Host.csproj src/Wiki.Host/
|
||||
RUN dotnet restore src/Wiki.Host/Wiki.Host.csproj
|
||||
|
||||
COPY . .
|
||||
# No --no-restore: publishing a Blazor Web App with it skips the static-asset
|
||||
# setup and blazor.web.js never lands in the output (dotnet/aspnetcore#65291).
|
||||
RUN dotnet publish src/Wiki.Host/Wiki.Host.csproj -c Release -o /app/publish /p:UseAppHost=false
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/publish ./
|
||||
|
||||
EXPOSE 8080
|
||||
ENV ASPNETCORE_URLS=http://0.0.0.0:8080 \
|
||||
DOTNET_RUNNING_IN_CONTAINER=true \
|
||||
DOTNET_NOLOGO=true
|
||||
|
||||
# Index snapshots and the catalog live here; a mounted volume in the cluster.
|
||||
RUN mkdir -p /data
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["dotnet", "Wiki.Host.dll"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Wiki
|
||||
|
||||
Web interface and agent API for the Notes vault (`k.renner/notes`), running in the homelab
|
||||
cluster. Design: `kubernetes/PROPOSAL-wiki-web-api.md` in the homelab repository.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
src/Wiki.Core domain types, ports, vault parsing — references nothing
|
||||
src/Wiki.Application use cases (sync, rank, status, …) — depends on Core only
|
||||
src/Wiki.Adapters.* one project per external thing, each references Core only
|
||||
src/Wiki.Host ASP.NET Core API + MudBlazor UI, composition root
|
||||
test/Wiki.Contracts one abstract suite per port; every adapter passes its suite
|
||||
```
|
||||
|
||||
Adapters are selected in `appsettings.json` under `Wiki:`; secrets come from environment
|
||||
variables (`Wiki__NoteStore__Gitea__Token`, `Wiki__Webhook__Secret`, `Auth__ApiKeys__<name>`).
|
||||
|
||||
## Run
|
||||
|
||||
```
|
||||
dotnet test Wiki.slnx
|
||||
dotnet run --project src/Wiki.Host
|
||||
```
|
||||
|
||||
Endpoints: `/health` (liveness), `/ready` (readiness), `/api/v1/status`, `POST /rank`,
|
||||
`POST /webhook`.
|
||||
@@ -0,0 +1,235 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.FileIndex;
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IDocumentCatalog"/> that keeps everything in memory and persists a single
|
||||
/// <c>catalog.json</c> under the configured directory. The link graph is rebuilt from
|
||||
/// <see cref="WikiLinks.Extract"/> on every upsert; only <see cref="DocumentKind.Wiki"/> documents
|
||||
/// contribute links and tags.
|
||||
/// </summary>
|
||||
public sealed class FileDocumentCatalog : IDocumentCatalog
|
||||
{
|
||||
private readonly string _path;
|
||||
private readonly Lock _gate = new();
|
||||
private readonly Dictionary<(DocumentKind Kind, string Slug), Document> _documents = [];
|
||||
private readonly Dictionary<string, List<Link>> _outgoing = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<Link>> _backlinks = new(StringComparer.Ordinal);
|
||||
private CatalogState _state = new(null, null);
|
||||
private bool _dirty;
|
||||
private string? _resetDetail;
|
||||
|
||||
public string Name => "file-catalog";
|
||||
|
||||
public FileDocumentCatalog(IOptions<FileIndexOptions> options)
|
||||
{
|
||||
Directory.CreateDirectory(options.Value.Directory);
|
||||
_path = Path.Combine(options.Value.Directory, "catalog.json");
|
||||
Load();
|
||||
}
|
||||
|
||||
private void Load()
|
||||
{
|
||||
if (!File.Exists(_path)) return;
|
||||
try
|
||||
{
|
||||
var json = File.ReadAllText(_path);
|
||||
var model = JsonSerializer.Deserialize<CatalogFileModel>(json);
|
||||
if (model is null) return;
|
||||
_state = new CatalogState(model.Commit, model.SyncedAt);
|
||||
foreach (var document in model.Documents)
|
||||
{
|
||||
_documents[(document.Kind, document.Slug)] = document;
|
||||
if (document.Kind == DocumentKind.Wiki) IndexLinks(document);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or IOException)
|
||||
{
|
||||
_resetDetail = $"catalog.json was unreadable ({ex.GetType().Name}); the catalog was reset to empty.";
|
||||
_documents.Clear();
|
||||
_outgoing.Clear();
|
||||
_backlinks.Clear();
|
||||
_state = new CatalogState(null, null);
|
||||
}
|
||||
}
|
||||
|
||||
private void IndexLinks(Document document)
|
||||
{
|
||||
var links = WikiLinks.Extract(document);
|
||||
_outgoing[document.Slug] = [.. links];
|
||||
foreach (var link in links)
|
||||
{
|
||||
if (!_backlinks.TryGetValue(link.ToSlug, out var incoming))
|
||||
{
|
||||
incoming = [];
|
||||
_backlinks[link.ToSlug] = incoming;
|
||||
}
|
||||
incoming.Add(link);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveLinksFrom(string slug)
|
||||
{
|
||||
if (!_outgoing.Remove(slug, out var links)) return;
|
||||
foreach (var link in links)
|
||||
{
|
||||
if (_backlinks.TryGetValue(link.ToSlug, out var incoming))
|
||||
{
|
||||
incoming.RemoveAll(l => l.FromSlug == slug);
|
||||
if (incoming.Count == 0) _backlinks.Remove(link.ToSlug);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Task<CatalogState> GetStateAsync(CancellationToken ct)
|
||||
{
|
||||
lock (_gate) return Task.FromResult(_state);
|
||||
}
|
||||
|
||||
public Task SetStateAsync(CatalogState state, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_state = state;
|
||||
_dirty = true;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(IReadOnlyList<Document> documents, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (var document in documents)
|
||||
{
|
||||
var key = (document.Kind, document.Slug);
|
||||
if (_documents.ContainsKey(key) && document.Kind == DocumentKind.Wiki)
|
||||
RemoveLinksFrom(document.Slug);
|
||||
_documents[key] = document;
|
||||
if (document.Kind == DocumentKind.Wiki) IndexLinks(document);
|
||||
}
|
||||
_dirty = true;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task RemoveAsync(IReadOnlyList<DocumentRef> references, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (var reference in references)
|
||||
{
|
||||
_documents.Remove((reference.Kind, reference.Slug));
|
||||
if (reference.Kind == DocumentKind.Wiki) RemoveLinksFrom(reference.Slug);
|
||||
}
|
||||
_dirty = true;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<Document?> GetAsync(DocumentKind kind, string slug, CancellationToken ct)
|
||||
{
|
||||
lock (_gate) return Task.FromResult(_documents.GetValueOrDefault((kind, slug)));
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DocumentRef>> ListAsync(DocumentKind kind, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
IReadOnlyList<DocumentRef> refs = _documents.Values.Where(d => d.Kind == kind).Select(d => d.Ref).ToList();
|
||||
return Task.FromResult(refs);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Document>> ListDocumentsAsync(DocumentKind kind, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
IReadOnlyList<Document> docs = _documents.Values.Where(d => d.Kind == kind).ToList();
|
||||
return Task.FromResult(docs);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Link>> BacklinksAsync(string slug, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
IReadOnlyList<Link> links = _backlinks.TryGetValue(slug, out var incoming) ? [.. incoming] : [];
|
||||
return Task.FromResult(links);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Link>> OutgoingLinksAsync(string slug, CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
IReadOnlyList<Link> links = _outgoing.TryGetValue(slug, out var outgoing) ? [.. outgoing] : [];
|
||||
return Task.FromResult(links);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IReadOnlyDictionary<string, int>> TagCountsAsync(CancellationToken ct)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var counts = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
foreach (var document in _documents.Values.Where(d => d.Kind == DocumentKind.Wiki))
|
||||
{
|
||||
foreach (var tag in document.Tags)
|
||||
counts[tag] = counts.GetValueOrDefault(tag) + 1;
|
||||
}
|
||||
return Task.FromResult<IReadOnlyDictionary<string, int>>(counts);
|
||||
}
|
||||
}
|
||||
|
||||
public Task FlushAsync(CancellationToken ct)
|
||||
{
|
||||
CatalogFileModel model;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_dirty) return Task.CompletedTask;
|
||||
model = new CatalogFileModel
|
||||
{
|
||||
Commit = _state.Commit,
|
||||
SyncedAt = _state.SyncedAt,
|
||||
Documents = [.. _documents.Values],
|
||||
};
|
||||
_dirty = false;
|
||||
}
|
||||
|
||||
var tmp = _path + ".tmp";
|
||||
File.WriteAllText(tmp, JsonSerializer.Serialize(model));
|
||||
File.Move(tmp, _path, overwrite: true);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var directory = Path.GetDirectoryName(_path)!;
|
||||
Directory.CreateDirectory(directory);
|
||||
var probe = Path.Combine(directory, $".health-{Guid.NewGuid():N}");
|
||||
File.WriteAllText(probe, string.Empty);
|
||||
File.Delete(probe);
|
||||
return Task.FromResult(new AdapterHealth(Name, true, _resetDetail, sw.Elapsed, DateTimeOffset.UtcNow));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return Task.FromResult(new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CatalogFileModel
|
||||
{
|
||||
public string? Commit { get; set; }
|
||||
public DateTimeOffset? SyncedAt { get; set; }
|
||||
public List<Document> Documents { get; set; } = [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Wiki.Adapters.FileIndex;
|
||||
|
||||
/// <summary>Where the file-backed adapters keep their state. Shared by <see cref="FileVectorIndex"/> and <see cref="FileDocumentCatalog"/>.</summary>
|
||||
public sealed class FileIndexOptions
|
||||
{
|
||||
/// <summary>Root directory. The vector index writes under <c>index/</c>; the catalog writes <c>catalog.json</c> directly inside it.</summary>
|
||||
public required string Directory { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
using System.Diagnostics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.FileIndex;
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IVectorIndex"/> that keeps everything in memory and persists one binary file per
|
||||
/// model under <c><Directory>/index/</c>. Queries are frequent; upserts are batched, so a
|
||||
/// <see cref="ReaderWriterLockSlim"/> guards the in-memory state rather than a single exclusive lock.
|
||||
/// </summary>
|
||||
public sealed class FileVectorIndex : IVectorIndex, IDisposable
|
||||
{
|
||||
private const float NormTolerance = 1e-3f;
|
||||
|
||||
private readonly string _indexDirectory;
|
||||
private readonly ReaderWriterLockSlim _lock = new(LockRecursionPolicy.NoRecursion);
|
||||
private readonly Dictionary<string, Dictionary<string, Entry>> _models = new(StringComparer.Ordinal);
|
||||
private readonly HashSet<string> _dirtyModels = new(StringComparer.Ordinal);
|
||||
private readonly List<string> _resetNotes = [];
|
||||
|
||||
public string Name => "file-index";
|
||||
|
||||
public FileVectorIndex(IOptions<FileIndexOptions> options)
|
||||
{
|
||||
_indexDirectory = Path.Combine(options.Value.Directory, "index");
|
||||
Directory.CreateDirectory(_indexDirectory);
|
||||
Load();
|
||||
}
|
||||
|
||||
private void Load()
|
||||
{
|
||||
if (!Directory.Exists(_indexDirectory)) return;
|
||||
|
||||
foreach (var path in Directory.EnumerateFiles(_indexDirectory, "*.bin"))
|
||||
{
|
||||
try
|
||||
{
|
||||
using var stream = File.OpenRead(path);
|
||||
using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: false);
|
||||
var modelId = reader.ReadString();
|
||||
var count = reader.ReadInt32();
|
||||
var entries = new Dictionary<string, Entry>(count, StringComparer.Ordinal);
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var slug = reader.ReadString();
|
||||
var hash = reader.ReadString();
|
||||
var dimension = reader.ReadInt32();
|
||||
var vector = new float[dimension];
|
||||
for (var d = 0; d < dimension; d++) vector[d] = reader.ReadSingle();
|
||||
entries[slug] = new Entry(hash, vector);
|
||||
}
|
||||
_models[modelId] = entries;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or EndOfStreamException or ArgumentOutOfRangeException or OutOfMemoryException or FormatException)
|
||||
{
|
||||
_resetNotes.Add($"{Path.GetFileName(path)} was unreadable ({ex.GetType().Name}); that model's index was reset to empty.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IndexState> DescribeAsync(string modelId, CancellationToken ct)
|
||||
{
|
||||
_lock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
var hashes = _models.TryGetValue(modelId, out var entries)
|
||||
? entries.ToDictionary(kv => kv.Key, kv => kv.Value.Hash, StringComparer.Ordinal)
|
||||
: new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
return Task.FromResult(new IndexState(modelId, hashes));
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitReadLock();
|
||||
}
|
||||
}
|
||||
|
||||
public Task UpsertAsync(IReadOnlyList<Embedding> embeddings, CancellationToken ct)
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
foreach (var embedding in embeddings)
|
||||
{
|
||||
if (!_models.TryGetValue(embedding.ModelId, out var entries))
|
||||
{
|
||||
entries = new Dictionary<string, Entry>(StringComparer.Ordinal);
|
||||
_models[embedding.ModelId] = entries;
|
||||
}
|
||||
entries[embedding.Slug] = new Entry(embedding.ContentHash, embedding.Vector);
|
||||
_dirtyModels.Add(embedding.ModelId);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task RemoveAsync(string modelId, IReadOnlyList<string> slugs, CancellationToken ct)
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
if (_models.TryGetValue(modelId, out var entries))
|
||||
{
|
||||
foreach (var slug in slugs) entries.Remove(slug);
|
||||
_dirtyModels.Add(modelId);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<RankedHit>> QueryAsync(string modelId, float[] query, int k, CancellationToken ct)
|
||||
{
|
||||
_lock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
if (k <= 0 || !_models.TryGetValue(modelId, out var entries) || entries.Count == 0)
|
||||
return Task.FromResult<IReadOnlyList<RankedHit>>([]);
|
||||
|
||||
IReadOnlyList<RankedHit> hits = entries
|
||||
.Select(kv => new RankedHit(kv.Key, Cosine(query, kv.Value.Vector)))
|
||||
.OrderByDescending(h => h.Score)
|
||||
.Take(k)
|
||||
.ToList();
|
||||
return Task.FromResult(hits);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitReadLock();
|
||||
}
|
||||
}
|
||||
|
||||
public Task FlushAsync(CancellationToken ct)
|
||||
{
|
||||
Dictionary<string, Dictionary<string, Entry>> toWrite;
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
if (_dirtyModels.Count == 0) return Task.CompletedTask;
|
||||
toWrite = new Dictionary<string, Dictionary<string, Entry>>(_dirtyModels.Count, StringComparer.Ordinal);
|
||||
foreach (var modelId in _dirtyModels)
|
||||
toWrite[modelId] = new Dictionary<string, Entry>(_models.TryGetValue(modelId, out var e) ? e : [], StringComparer.Ordinal);
|
||||
_dirtyModels.Clear();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
|
||||
foreach (var (modelId, entries) in toWrite)
|
||||
WriteModel(modelId, entries);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void WriteModel(string modelId, Dictionary<string, Entry> entries)
|
||||
{
|
||||
var path = Path.Combine(_indexDirectory, FileNameFor(modelId));
|
||||
var tmp = path + ".tmp";
|
||||
using (var stream = File.Create(tmp))
|
||||
using (var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: false))
|
||||
{
|
||||
writer.Write(modelId);
|
||||
writer.Write(entries.Count);
|
||||
foreach (var (slug, entry) in entries)
|
||||
{
|
||||
writer.Write(slug);
|
||||
writer.Write(entry.Hash);
|
||||
writer.Write(entry.Vector.Length);
|
||||
foreach (var component in entry.Vector) writer.Write(component);
|
||||
}
|
||||
}
|
||||
File.Move(tmp, path, overwrite: true);
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_indexDirectory);
|
||||
var probe = Path.Combine(_indexDirectory, $".health-{Guid.NewGuid():N}");
|
||||
File.WriteAllText(probe, string.Empty);
|
||||
File.Delete(probe);
|
||||
var detail = _resetNotes.Count == 0 ? null : string.Join(' ', _resetNotes);
|
||||
return Task.FromResult(new AdapterHealth(Name, true, detail, sw.Elapsed, DateTimeOffset.UtcNow));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return Task.FromResult(new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Filesystem-safe and collision-resistant: readable prefix plus a content hash of the real id.</summary>
|
||||
private static string FileNameFor(string modelId)
|
||||
{
|
||||
var sanitized = new string(modelId.Select(c => char.IsLetterOrDigit(c) || c is '-' or '_' or '.' ? c : '_').ToArray());
|
||||
var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(modelId)))[..8];
|
||||
return $"{sanitized}-{hash}.bin";
|
||||
}
|
||||
|
||||
/// <summary>Cosine similarity. Vectors are unit length by contract; norms are still checked cheaply in case they are not.</summary>
|
||||
private static float Cosine(float[] a, float[] b)
|
||||
{
|
||||
var length = Math.Min(a.Length, b.Length);
|
||||
float dot = 0, normA = 0, normB = 0;
|
||||
for (var i = 0; i < length; i++)
|
||||
{
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
normA = MathF.Sqrt(normA);
|
||||
normB = MathF.Sqrt(normB);
|
||||
if (MathF.Abs(normA - 1f) <= NormTolerance && MathF.Abs(normB - 1f) <= NormTolerance)
|
||||
return dot;
|
||||
var denom = normA * normB;
|
||||
return denom > 0 ? dot / denom : 0f;
|
||||
}
|
||||
|
||||
public void Dispose() => _lock.Dispose();
|
||||
|
||||
private sealed record Entry(string Hash, float[] Vector);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.FileIndex;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>Registers <see cref="FileVectorIndex"/> and <see cref="FileDocumentCatalog"/> as singletons sharing one <see cref="FileIndexOptions.Directory"/>.</summary>
|
||||
public static IServiceCollection AddFileIndexAdapters(this IServiceCollection services, Action<FileIndexOptions> configure)
|
||||
{
|
||||
services.Configure(configure);
|
||||
|
||||
services.AddSingleton<FileVectorIndex>();
|
||||
services.AddSingleton<IVectorIndex>(sp => sp.GetRequiredService<FileVectorIndex>());
|
||||
|
||||
services.AddSingleton<FileDocumentCatalog>();
|
||||
services.AddSingleton<IDocumentCatalog>(sp => sp.GetRequiredService<FileDocumentCatalog>());
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,11 @@
|
||||
<ProjectReference Include="..\Wiki.Core\Wiki.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
// Shapes of the Gitea API responses actually used. Verified against a live instance
|
||||
// (GET /repos/{o}/{r}/branches/{b}, GET /repos/{o}/{r}/git/trees/{sha}) rather than the swagger doc.
|
||||
|
||||
internal sealed class BranchResponse
|
||||
{
|
||||
[JsonPropertyName("commit")]
|
||||
public BranchCommit Commit { get; set; } = new();
|
||||
}
|
||||
|
||||
internal sealed class BranchCommit
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("timestamp")]
|
||||
public DateTimeOffset? Timestamp { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class TreeResponse
|
||||
{
|
||||
[JsonPropertyName("tree")]
|
||||
public List<TreeEntry>? Tree { get; set; }
|
||||
|
||||
[JsonPropertyName("total_count")]
|
||||
public int TotalCount { get; set; }
|
||||
|
||||
[JsonPropertyName("truncated")]
|
||||
public bool Truncated { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class TreeEntry
|
||||
{
|
||||
[JsonPropertyName("path")]
|
||||
public string Path { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("sha")]
|
||||
public string Sha { get; set; } = "";
|
||||
}
|
||||
|
||||
/// <summary>The subset of Gitea's "create file" response (POST .../contents/{path}) that a fact sink needs.</summary>
|
||||
internal sealed class ContentsCreateResponse
|
||||
{
|
||||
[JsonPropertyName("commit")]
|
||||
public ContentsCommit Commit { get; set; } = new();
|
||||
}
|
||||
|
||||
internal sealed class ContentsCommit
|
||||
{
|
||||
[JsonPropertyName("sha")]
|
||||
public string Sha { get; set; } = "";
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
/// <summary>Deposits a fact as a new file under raw/ in Gitea. Never edits an existing file: a
|
||||
/// title that would collide with an existing path at HEAD gets a numbered suffix instead.</summary>
|
||||
public sealed class GiteaFactSink : IFactSink
|
||||
{
|
||||
private const int MaxSlugLength = 80;
|
||||
private const int MaxSuffixAttempts = 1000;
|
||||
|
||||
private readonly HttpClient _http;
|
||||
private readonly GiteaOptions _options;
|
||||
|
||||
public string Name => "gitea";
|
||||
|
||||
public GiteaFactSink(HttpClient httpClient, IOptions<GiteaOptions> options)
|
||||
{
|
||||
_http = httpClient;
|
||||
_options = options.Value;
|
||||
GiteaHttp.Configure(_http, _options);
|
||||
}
|
||||
|
||||
public async Task<FactReceipt> AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct)
|
||||
{
|
||||
var errors = fact.Validate();
|
||||
if (errors.Count > 0)
|
||||
throw new ArgumentException($"Invalid raw source: {string.Join("; ", errors)}", nameof(fact));
|
||||
|
||||
var datePrefix = now.UtcDateTime.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
|
||||
var slug = Slugify(fact.Title);
|
||||
var path = await ReservePathAsync(datePrefix, slug, ct).ConfigureAwait(false);
|
||||
var text = BuildFile(fact, by, now, datePrefix);
|
||||
|
||||
var repo = GiteaUrls.EncodeSegments(_options.Repository);
|
||||
var encodedPath = GiteaUrls.EncodeSegments(path);
|
||||
var request = new
|
||||
{
|
||||
content = Convert.ToBase64String(Encoding.UTF8.GetBytes(text)),
|
||||
message = $"raw: {fact.Title}",
|
||||
branch = _options.Branch,
|
||||
};
|
||||
using var response = await _http.PostAsJsonAsync($"api/v1/repos/{repo}/contents/{encodedPath}", request, GiteaHttp.Json, ct)
|
||||
.ConfigureAwait(false);
|
||||
await GiteaHttp.EnsureSuccessAsync(response, ct).ConfigureAwait(false);
|
||||
var created = await response.Content.ReadFromJsonAsync<ContentsCreateResponse>(GiteaHttp.Json, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException("Gitea returned an empty body creating a file.");
|
||||
return new FactReceipt(path, created.Commit.Sha, now);
|
||||
}
|
||||
|
||||
public async Task<AdapterHealth> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var repo = GiteaUrls.EncodeSegments(_options.Repository);
|
||||
var branch = Uri.EscapeDataString(_options.Branch);
|
||||
var head = await GiteaHttp.GetJsonAsync<BranchResponse>(_http, $"api/v1/repos/{repo}/branches/{branch}", ct)
|
||||
.ConfigureAwait(false);
|
||||
return new AdapterHealth(Name, true, $"HEAD {head.Commit.Id}", sw.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The candidate path if free at HEAD, else the same with -2, -3, ... appended until one is free.</summary>
|
||||
private async Task<string> ReservePathAsync(string datePrefix, string slug, CancellationToken ct)
|
||||
{
|
||||
var candidate = $"raw/{datePrefix}-{slug}.md";
|
||||
for (var attempt = 2; await ExistsAtHeadAsync(candidate, ct).ConfigureAwait(false); attempt++)
|
||||
{
|
||||
if (attempt > MaxSuffixAttempts)
|
||||
throw new InvalidOperationException($"Gitea has {MaxSuffixAttempts}+ raw files named like {datePrefix}-{slug}; giving up.");
|
||||
candidate = $"raw/{datePrefix}-{slug}-{attempt}.md";
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private async Task<bool> ExistsAtHeadAsync(string path, CancellationToken ct)
|
||||
{
|
||||
var repo = GiteaUrls.EncodeSegments(_options.Repository);
|
||||
var encodedPath = GiteaUrls.EncodeSegments(path);
|
||||
var branch = Uri.EscapeDataString(_options.Branch);
|
||||
using var response = await _http.GetAsync($"api/v1/repos/{repo}/contents/{encodedPath}?ref={branch}", ct).ConfigureAwait(false);
|
||||
if (response.StatusCode == HttpStatusCode.NotFound) return false;
|
||||
await GiteaHttp.EnsureSuccessAsync(response, ct).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string BuildFile(RawSource fact, Identity by, DateTimeOffset now, string capturedDate)
|
||||
{
|
||||
var body = fact.Content;
|
||||
if (!body.EndsWith('\n')) body += "\n";
|
||||
|
||||
var sb = new StringBuilder();
|
||||
sb.Append("---\n");
|
||||
sb.Append("title: ").Append(YamlScalar(fact.Title)).Append('\n');
|
||||
sb.Append("type: raw\n");
|
||||
sb.Append("source: ").Append(YamlScalar(fact.Source)).Append('\n');
|
||||
sb.Append("captured: ").Append(capturedDate).Append('\n');
|
||||
sb.Append("captured_by: ").Append(by.Name).Append('\n');
|
||||
sb.Append("observed: ").Append(fact.ObservedAt.ToString("yyyy-MM-ddTHH:mm:ssK", CultureInfo.InvariantCulture)).Append('\n');
|
||||
sb.Append("tags: [").Append(string.Join(", ", fact.Tags)).Append("]\n");
|
||||
sb.Append("---\n");
|
||||
sb.Append('\n');
|
||||
sb.Append(body);
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>Quotes a scalar that would otherwise be ambiguous or wrong in flow-less YAML: a colon
|
||||
/// or a '#' can start a mapping value or a comment mid-line, and a leading quote needs escaping.</summary>
|
||||
private static string YamlScalar(string value)
|
||||
{
|
||||
if (!value.Contains(':') && !value.Contains('#') && !value.StartsWith('"') && !value.StartsWith('\''))
|
||||
return value;
|
||||
return $"\"{value.Replace("\"", "\\\"")}\"";
|
||||
}
|
||||
|
||||
private static string Slugify(string title)
|
||||
{
|
||||
var sb = new StringBuilder(title.Length);
|
||||
foreach (var ch in title.ToLowerInvariant())
|
||||
{
|
||||
if (ch is (>= 'a' and <= 'z') or (>= '0' and <= '9')) sb.Append(ch);
|
||||
else sb.Append('-');
|
||||
}
|
||||
|
||||
var collapsed = new StringBuilder(sb.Length);
|
||||
var lastWasDash = false;
|
||||
foreach (var ch in sb.ToString())
|
||||
{
|
||||
if (ch == '-')
|
||||
{
|
||||
if (lastWasDash) continue;
|
||||
lastWasDash = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
lastWasDash = false;
|
||||
}
|
||||
collapsed.Append(ch);
|
||||
}
|
||||
|
||||
var slug = collapsed.ToString().Trim('-');
|
||||
if (slug.Length > MaxSlugLength) slug = slug[..MaxSlugLength].Trim('-');
|
||||
return slug.Length == 0 ? "untitled" : slug;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
/// <summary>HTTP plumbing shared by <see cref="GiteaNoteStore"/> and <see cref="GiteaFactSink"/>:
|
||||
/// client setup, JSON decoding and turning a failed response into a message worth reading.</summary>
|
||||
internal static class GiteaHttp
|
||||
{
|
||||
public static readonly JsonSerializerOptions Json = new();
|
||||
|
||||
private const int ErrorBodyPreviewLength = 300;
|
||||
|
||||
/// <summary>Points the client at the configured Gitea and carries the token, unless the caller
|
||||
/// (a test, typically) has already set these up on the HttpClient it handed in.</summary>
|
||||
public static void Configure(HttpClient client, GiteaOptions options)
|
||||
{
|
||||
client.BaseAddress ??= new Uri(options.BaseUrl, UriKind.Absolute);
|
||||
if (client.DefaultRequestHeaders.Authorization is null && !string.IsNullOrEmpty(options.Token))
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("token", options.Token);
|
||||
}
|
||||
|
||||
public static async Task<T> GetJsonAsync<T>(HttpClient client, string url, CancellationToken ct)
|
||||
{
|
||||
using var response = await client.GetAsync(url, ct).ConfigureAwait(false);
|
||||
await EnsureSuccessAsync(response, ct).ConfigureAwait(false);
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
|
||||
return await JsonSerializer.DeserializeAsync<T>(stream, Json, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Gitea returned an empty body for {url}.");
|
||||
}
|
||||
|
||||
public static async Task EnsureSuccessAsync(HttpResponseMessage response, CancellationToken ct)
|
||||
{
|
||||
if (response.IsSuccessStatusCode) return;
|
||||
var body = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
|
||||
if (body.Length > ErrorBodyPreviewLength) body = body[..ErrorBodyPreviewLength];
|
||||
throw new HttpRequestException(
|
||||
$"Gitea {(int)response.StatusCode} {response.ReasonPhrase} for {response.RequestMessage?.RequestUri}: {body}",
|
||||
null,
|
||||
response.StatusCode);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
/// <summary>Reads the vault straight from a Gitea repository: HEAD, a recursive tree listing and raw
|
||||
/// file contents, all at the commit the listing was taken from.</summary>
|
||||
public sealed class GiteaNoteStore : INoteStore
|
||||
{
|
||||
// Gitea answers "tree": null past the last page, and may mark the last page truncated=true even
|
||||
// though nothing follows; stop on a short/empty page or once total_count is reached either way.
|
||||
private const int PageSize = 1000;
|
||||
private const int MaxPages = 50;
|
||||
|
||||
private readonly HttpClient _http;
|
||||
private readonly GiteaOptions _options;
|
||||
|
||||
public string Name => "gitea";
|
||||
|
||||
public GiteaNoteStore(HttpClient httpClient, IOptions<GiteaOptions> options)
|
||||
{
|
||||
_http = httpClient;
|
||||
_options = options.Value;
|
||||
GiteaHttp.Configure(_http, _options);
|
||||
}
|
||||
|
||||
public async Task<StoreHead> GetHeadAsync(CancellationToken ct)
|
||||
{
|
||||
var branch = await FetchBranchAsync(ct).ConfigureAwait(false);
|
||||
return new StoreHead(branch.Commit.Id, branch.Commit.Timestamp);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<DocumentRef>> ListAsync(string commit, IReadOnlySet<DocumentKind> kinds, CancellationToken ct)
|
||||
{
|
||||
var repo = GiteaUrls.EncodeSegments(_options.Repository);
|
||||
var refs = new List<DocumentRef>();
|
||||
var page = 1;
|
||||
var seen = 0;
|
||||
while (true)
|
||||
{
|
||||
var url = $"api/v1/repos/{repo}/git/trees/{Uri.EscapeDataString(commit)}?recursive=true&per_page={PageSize}&page={page}";
|
||||
var tree = await GiteaHttp.GetJsonAsync<TreeResponse>(_http, url, ct).ConfigureAwait(false);
|
||||
var entries = tree.Tree ?? [];
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
if (entry.Type != "blob") continue;
|
||||
if (DocumentParser.Classify(entry.Path) is not { } classified) continue;
|
||||
if (!kinds.Contains(classified.Kind)) continue;
|
||||
refs.Add(new DocumentRef(classified.Kind, classified.Slug, entry.Path, entry.Sha));
|
||||
}
|
||||
seen += entries.Count;
|
||||
// total_count is the reliable signal (confirmed live: Gitea reports it regardless of the
|
||||
// per_page requested); a short page only means "done" when total_count isn't available,
|
||||
// since a server may cap the effective page size below what was asked for.
|
||||
var done = entries.Count == 0
|
||||
|| (tree.TotalCount > 0 ? seen >= tree.TotalCount : entries.Count < PageSize);
|
||||
if (done) break;
|
||||
page++;
|
||||
if (page > MaxPages) throw new InvalidOperationException("Gitea tree paging exceeded the safety cap.");
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
public async Task<string> ReadAsync(DocumentRef reference, string commit, CancellationToken ct)
|
||||
{
|
||||
var repo = GiteaUrls.EncodeSegments(_options.Repository);
|
||||
var path = GiteaUrls.EncodeSegments(reference.Path);
|
||||
var url = $"api/v1/repos/{repo}/raw/{path}?ref={Uri.EscapeDataString(commit)}";
|
||||
using var response = await _http.GetAsync(url, ct).ConfigureAwait(false);
|
||||
await GiteaHttp.EnsureSuccessAsync(response, ct).ConfigureAwait(false);
|
||||
return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<AdapterHealth> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var branch = await FetchBranchAsync(ct).ConfigureAwait(false);
|
||||
return new AdapterHealth(Name, true, $"HEAD {branch.Commit.Id}", sw.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new AdapterHealth(Name, false, ex.Message, sw.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
}
|
||||
|
||||
private Task<BranchResponse> FetchBranchAsync(CancellationToken ct)
|
||||
{
|
||||
var repo = GiteaUrls.EncodeSegments(_options.Repository);
|
||||
var branch = Uri.EscapeDataString(_options.Branch);
|
||||
return GiteaHttp.GetJsonAsync<BranchResponse>(_http, $"api/v1/repos/{repo}/branches/{branch}", ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
/// <summary>How to reach the notes repository on Gitea.</summary>
|
||||
public sealed class GiteaOptions
|
||||
{
|
||||
public string BaseUrl { get; set; } = "https://gitea.renner.uno";
|
||||
public string Repository { get; set; } = "k.renner/notes";
|
||||
public string Branch { get; set; } = "main";
|
||||
public string Token { get; set; } = "";
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
/// <summary>Path building shared by the store and the sink: repo/branch/path segments are each
|
||||
/// percent-encoded on their own so a literal '/' between segments survives, matching Gitea's routing.</summary>
|
||||
internal static class GiteaUrls
|
||||
{
|
||||
public static string EncodeSegments(string value) =>
|
||||
string.Join('/', value.Split('/').Select(Uri.EscapeDataString));
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>Registers the Gitea-backed <see cref="INoteStore"/> and <see cref="IFactSink"/>, each
|
||||
/// with its own managed HttpClient from <see cref="IHttpClientFactory"/>. Reads retry transient
|
||||
/// failures; pooled connections are recycled so a stale keep-alive is not reused for long.</summary>
|
||||
public static IServiceCollection AddGiteaAdapters(this IServiceCollection services, Action<GiteaOptions> configure)
|
||||
{
|
||||
services.Configure(configure);
|
||||
services.AddHttpClient<GiteaNoteStore>()
|
||||
.AddHttpMessageHandler(() => new TransientRetryHandler())
|
||||
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(2) });
|
||||
services.AddHttpClient<GiteaFactSink>()
|
||||
.AddHttpMessageHandler(() => new TransientRetryHandler())
|
||||
.ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(2) });
|
||||
services.AddTransient<INoteStore>(sp => sp.GetRequiredService<GiteaNoteStore>());
|
||||
services.AddTransient<IFactSink>(sp => sp.GetRequiredService<GiteaFactSink>());
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System.Net;
|
||||
|
||||
namespace Wiki.Adapters.Gitea;
|
||||
|
||||
/// <summary>
|
||||
/// Retries idempotent requests (GET, HEAD) on transport failures and 5xx responses with a short
|
||||
/// backoff. A sync reads thousands of documents over one keep-alive connection; Gitea behind
|
||||
/// Traefik resets such a connection now and then, and one reset must not abort the whole run.
|
||||
/// Non-idempotent requests (the fact sink's POST) are never retried here.
|
||||
/// </summary>
|
||||
public sealed class TransientRetryHandler(int attempts = 3, TimeSpan? firstDelay = null, Func<TimeSpan, CancellationToken, Task>? delay = null) : DelegatingHandler
|
||||
{
|
||||
private readonly TimeSpan _firstDelay = firstDelay ?? TimeSpan.FromMilliseconds(500);
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay = delay ?? Task.Delay;
|
||||
|
||||
public int Attempts { get; } = Math.Max(1, attempts);
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
var idempotent = request.Method == HttpMethod.Get || request.Method == HttpMethod.Head;
|
||||
var wait = _firstDelay;
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
HttpResponseMessage? response = null;
|
||||
try
|
||||
{
|
||||
response = await base.SendAsync(request, ct).ConfigureAwait(false);
|
||||
if (!idempotent || attempt >= Attempts || !IsTransient(response.StatusCode))
|
||||
return response;
|
||||
}
|
||||
catch (HttpRequestException) when (idempotent && attempt < Attempts)
|
||||
{
|
||||
}
|
||||
catch (IOException) when (idempotent && attempt < Attempts)
|
||||
{
|
||||
}
|
||||
catch (TaskCanceledException) when (idempotent && attempt < Attempts && !ct.IsCancellationRequested)
|
||||
{
|
||||
// HttpClient timeout, not the caller cancelling
|
||||
}
|
||||
response?.Dispose();
|
||||
await _delay(wait, ct).ConfigureAwait(false);
|
||||
wait += wait;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTransient(HttpStatusCode status) =>
|
||||
status is HttpStatusCode.BadGateway or HttpStatusCode.ServiceUnavailable or HttpStatusCode.GatewayTimeout or HttpStatusCode.InternalServerError;
|
||||
}
|
||||
@@ -4,6 +4,11 @@
|
||||
<ProjectReference Include="..\Wiki.Core\Wiki.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.12" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace Wiki.Adapters.LlamaCpp;
|
||||
|
||||
/// <summary>Request body for POST /v1/embeddings.</summary>
|
||||
internal sealed record EmbeddingRequest(string Model, IReadOnlyList<string> Input);
|
||||
|
||||
/// <summary>Response body from POST /v1/embeddings.</summary>
|
||||
internal sealed record EmbeddingResponse(IReadOnlyList<EmbeddingRow> Data);
|
||||
|
||||
/// <summary>One row of an embeddings response; rows are not guaranteed to arrive in request order.</summary>
|
||||
internal sealed record EmbeddingRow(int Index, float[] Embedding);
|
||||
@@ -0,0 +1,126 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.LlamaCpp;
|
||||
|
||||
/// <summary>Embeds text via a llama.cpp server's OpenAI-compatible <c>/v1/embeddings</c> endpoint.</summary>
|
||||
public sealed class LlamaCppEmbeddingModel : IEmbeddingModel
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
private readonly HttpClient _http;
|
||||
private readonly LlamaCppEmbeddingOptions _options;
|
||||
|
||||
public LlamaCppEmbeddingModel(HttpClient http, LlamaCppEmbeddingOptions options)
|
||||
{
|
||||
_http = http;
|
||||
_options = options;
|
||||
}
|
||||
|
||||
public string ModelId => _options.ModelId;
|
||||
|
||||
public int Dimension => _options.Dimension;
|
||||
|
||||
public string Name => $"llamacpp-embeddings:{_options.ModelId}";
|
||||
|
||||
public async Task<IReadOnlyList<float[]>> EmbedAsync(IReadOnlyList<string> texts, CancellationToken ct)
|
||||
{
|
||||
if (texts.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
var result = new float[texts.Count][];
|
||||
for (var offset = 0; offset < texts.Count; offset += _options.BatchSize)
|
||||
{
|
||||
var batch = texts.Skip(offset).Take(_options.BatchSize).ToArray();
|
||||
var vectors = await EmbedBatchAsync(batch, ct).ConfigureAwait(false);
|
||||
for (var i = 0; i < vectors.Length; i++)
|
||||
{
|
||||
result[offset + i] = vectors[i];
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<AdapterHealth> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
using var response = await _http.GetAsync("health", ct).ConfigureAwait(false);
|
||||
if (response.StatusCode == HttpStatusCode.NotFound)
|
||||
{
|
||||
await EmbedBatchAsync(["health"], ct).ConfigureAwait(false);
|
||||
return new AdapterHealth(Name, true, "no /health endpoint; verified via /v1/embeddings", stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
return new AdapterHealth(Name, true, null, stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
return new AdapterHealth(Name, false, $"GET /health returned {(int)response.StatusCode}", stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException)
|
||||
{
|
||||
return new AdapterHealth(Name, false, ex.Message, stopwatch.Elapsed, DateTimeOffset.UtcNow);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<float[][]> EmbedBatchAsync(IReadOnlyList<string> batch, CancellationToken ct)
|
||||
{
|
||||
var request = new EmbeddingRequest(_options.Model, batch);
|
||||
var json = JsonSerializer.Serialize(request, JsonOptions);
|
||||
using var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
using var response = await _http.PostAsync("v1/embeddings", content, ct).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var body = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
|
||||
var payload = await JsonSerializer.DeserializeAsync<EmbeddingResponse>(body, JsonOptions, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException("llama.cpp embeddings response body was empty.");
|
||||
|
||||
var ordered = payload.Data.OrderBy(row => row.Index).ToArray();
|
||||
var vectors = new float[ordered.Length][];
|
||||
for (var i = 0; i < ordered.Length; i++)
|
||||
{
|
||||
vectors[i] = Normalize(ordered[i].Embedding);
|
||||
}
|
||||
|
||||
return vectors;
|
||||
}
|
||||
|
||||
private float[] Normalize(float[] vector)
|
||||
{
|
||||
if (vector.Length != _options.Dimension)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"llama.cpp returned an embedding of dimension {vector.Length}, expected {_options.Dimension}.");
|
||||
}
|
||||
|
||||
var sumSquares = 0.0;
|
||||
foreach (var x in vector)
|
||||
{
|
||||
sumSquares += (double)x * x;
|
||||
}
|
||||
|
||||
var norm = Math.Sqrt(sumSquares);
|
||||
if (norm == 0)
|
||||
{
|
||||
return vector;
|
||||
}
|
||||
|
||||
var result = new float[vector.Length];
|
||||
for (var i = 0; i < vector.Length; i++)
|
||||
{
|
||||
result[i] = (float)(vector[i] / norm);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
namespace Wiki.Adapters.LlamaCpp;
|
||||
|
||||
/// <summary>Configuration for one llama.cpp <c>/v1/embeddings</c> endpoint.</summary>
|
||||
public sealed class LlamaCppEmbeddingOptions
|
||||
{
|
||||
private string? _modelId;
|
||||
|
||||
/// <summary>Base URL of the llama-server, e.g. http://wiki-embed.ai.svc.cluster.local:8080.</summary>
|
||||
public string Url { get; set; } = "";
|
||||
|
||||
/// <summary>Model name sent in the request body, e.g. "bge-m3".</summary>
|
||||
public string Model { get; set; } = "";
|
||||
|
||||
/// <summary>Identifier used as the index namespace. Defaults to <see cref="Model"/>.</summary>
|
||||
public string ModelId
|
||||
{
|
||||
get => string.IsNullOrWhiteSpace(_modelId) ? Model : _modelId;
|
||||
set => _modelId = value;
|
||||
}
|
||||
|
||||
/// <summary>Length every returned embedding must have.</summary>
|
||||
public int Dimension { get; set; }
|
||||
|
||||
/// <summary>Requests are chunked to this many texts; llama-server's embedding endpoint chokes on larger batches for long notes.</summary>
|
||||
public int BatchSize { get; set; } = 2;
|
||||
|
||||
/// <summary>Optional "user:password" sent as HTTP Basic auth, for reaching the service through the Traefik BasicAuth ingress from outside the cluster.</summary>
|
||||
public string? BasicAuth { get; set; }
|
||||
|
||||
/// <summary>Per-request timeout.</summary>
|
||||
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(120);
|
||||
|
||||
internal void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Url))
|
||||
{
|
||||
throw new ArgumentException("Url must be set.", nameof(Url));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Model))
|
||||
{
|
||||
throw new ArgumentException("Model must be set.", nameof(Model));
|
||||
}
|
||||
|
||||
if (Dimension <= 0)
|
||||
{
|
||||
throw new ArgumentException("Dimension must be positive.", nameof(Dimension));
|
||||
}
|
||||
|
||||
if (BatchSize <= 0)
|
||||
{
|
||||
throw new ArgumentException("BatchSize must be positive.", nameof(BatchSize));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Adapters.LlamaCpp;
|
||||
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers an <see cref="IEmbeddingModel"/> backed by a llama.cpp server, keyed by <paramref name="name"/>.
|
||||
/// Several models may be registered under different names, each with its own <see cref="LlamaCppEmbeddingOptions"/>.
|
||||
/// </summary>
|
||||
public static IServiceCollection AddLlamaCppEmbeddings(
|
||||
this IServiceCollection services,
|
||||
string name,
|
||||
Action<LlamaCppEmbeddingOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(name);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
var options = new LlamaCppEmbeddingOptions();
|
||||
configure(options);
|
||||
options.Validate();
|
||||
|
||||
var httpClientName = $"llamacpp-embeddings:{name}";
|
||||
|
||||
services.AddHttpClient(httpClientName, client =>
|
||||
{
|
||||
client.BaseAddress = new Uri(options.Url.TrimEnd('/') + "/");
|
||||
client.Timeout = options.Timeout;
|
||||
|
||||
if (!string.IsNullOrEmpty(options.BasicAuth))
|
||||
{
|
||||
var token = Convert.ToBase64String(Encoding.UTF8.GetBytes(options.BasicAuth));
|
||||
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", token);
|
||||
}
|
||||
});
|
||||
|
||||
services.AddKeyedSingleton<IEmbeddingModel>(name, (sp, _) =>
|
||||
{
|
||||
var httpClientFactory = sp.GetRequiredService<IHttpClientFactory>();
|
||||
var client = httpClientFactory.CreateClient(httpClientName);
|
||||
return new LlamaCppEmbeddingModel(client, options);
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,11 @@
|
||||
<ProjectReference Include="..\Wiki.Core\Wiki.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.12" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Retrieval;
|
||||
|
||||
public sealed record RankedNote(string Slug, string Title, float Score);
|
||||
|
||||
/// <summary>Embed the question with the chosen model, query its index namespace, resolve titles.</summary>
|
||||
public sealed class RankUseCase(IVectorIndex index, IDocumentCatalog catalog)
|
||||
{
|
||||
public const int MaxK = 500;
|
||||
|
||||
public async Task<IReadOnlyList<RankedNote>> RankAsync(IEmbeddingModel model, string question, int k, CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(question)) throw new ArgumentException("question required", nameof(question));
|
||||
k = Math.Clamp(k, 1, MaxK);
|
||||
var vector = (await model.EmbedAsync([question.Trim()], ct))[0];
|
||||
var hits = await index.QueryAsync(model.ModelId, vector, k, ct);
|
||||
var rows = new List<RankedNote>(hits.Count);
|
||||
foreach (var hit in hits)
|
||||
{
|
||||
var doc = await catalog.GetAsync(DocumentKind.Wiki, hit.Slug, ct);
|
||||
rows.Add(new RankedNote(hit.Slug, doc?.Title ?? hit.Slug, hit.Score));
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
using Wiki.Application.Sync;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Status;
|
||||
|
||||
public sealed record IndexStatus(string ModelId, int Notes, bool Active);
|
||||
|
||||
public sealed record ServiceStatus(
|
||||
bool Ready,
|
||||
string? Commit,
|
||||
DateTimeOffset? SyncedAt,
|
||||
double? AgeSeconds,
|
||||
bool Syncing,
|
||||
string? LastError,
|
||||
int Notes,
|
||||
int RawQueue,
|
||||
int Outputs,
|
||||
IReadOnlyList<IndexStatus> Indexes,
|
||||
IReadOnlyList<AdapterHealth> Adapters);
|
||||
|
||||
/// <summary>
|
||||
/// Readiness is "the catalog has a commit and the active index has notes". Adapter checks are
|
||||
/// informational and never gate readiness.
|
||||
/// </summary>
|
||||
public sealed class StatusQuery(
|
||||
IDocumentCatalog catalog,
|
||||
IVectorIndex index,
|
||||
IReadOnlyList<IEmbeddingModel> embeddingModels,
|
||||
IEmbeddingModel activeModel,
|
||||
SyncCoordinator sync,
|
||||
IReadOnlyList<IAdapterHealth> adapters,
|
||||
TimeProvider clock)
|
||||
{
|
||||
public async Task<bool> IsReadyAsync(CancellationToken ct)
|
||||
{
|
||||
var state = await catalog.GetStateAsync(ct);
|
||||
if (state.Commit is null) return false;
|
||||
var active = await index.DescribeAsync(activeModel.ModelId, ct);
|
||||
return active.Count > 0;
|
||||
}
|
||||
|
||||
public async Task<ServiceStatus> GetAsync(bool includeAdapterChecks, CancellationToken ct)
|
||||
{
|
||||
var state = await catalog.GetStateAsync(ct);
|
||||
var indexes = new List<IndexStatus>();
|
||||
foreach (var m in embeddingModels)
|
||||
{
|
||||
var s = await index.DescribeAsync(m.ModelId, ct);
|
||||
indexes.Add(new IndexStatus(m.ModelId, s.Count, m.ModelId == activeModel.ModelId));
|
||||
}
|
||||
var checks = new List<AdapterHealth>();
|
||||
if (includeAdapterChecks)
|
||||
{
|
||||
foreach (var a in adapters)
|
||||
{
|
||||
try { checks.Add(await a.CheckAsync(ct)); }
|
||||
catch (Exception e) { checks.Add(new AdapterHealth(a.Name, false, e.Message, TimeSpan.Zero, clock.GetUtcNow())); }
|
||||
}
|
||||
}
|
||||
var now = clock.GetUtcNow();
|
||||
return new ServiceStatus(
|
||||
await IsReadyAsync(ct),
|
||||
state.Commit,
|
||||
state.SyncedAt,
|
||||
state.SyncedAt is { } t ? Math.Round((now - t).TotalSeconds) : null,
|
||||
sync.Syncing,
|
||||
sync.LastError,
|
||||
(await catalog.ListAsync(DocumentKind.Wiki, ct)).Count,
|
||||
(await catalog.ListAsync(DocumentKind.Raw, ct)).Count,
|
||||
(await catalog.ListAsync(DocumentKind.Output, ct)).Count,
|
||||
indexes,
|
||||
checks);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Wiki.Application.Sync;
|
||||
|
||||
/// <summary>
|
||||
/// One sync at a time; a caller arriving during a run waits and then runs its own, so a webhook
|
||||
/// that lands mid-sync is never lost. Keeps the last outcome for status. Failures are recorded
|
||||
/// and returned as null, never thrown to the trigger.
|
||||
/// </summary>
|
||||
public sealed class SyncCoordinator(SyncUseCase sync, TimeProvider clock, ILogger<SyncCoordinator> log)
|
||||
{
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
|
||||
public bool Syncing { get; private set; }
|
||||
public SyncSummary? LastSummary { get; private set; }
|
||||
public string? LastError { get; private set; }
|
||||
public DateTimeOffset? LastAttemptAt { get; private set; }
|
||||
|
||||
public async Task<SyncSummary?> RunAsync(bool force, CancellationToken ct)
|
||||
{
|
||||
await _gate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
Syncing = true;
|
||||
LastAttemptAt = clock.GetUtcNow();
|
||||
try
|
||||
{
|
||||
LastSummary = await sync.RunAsync(force, ct);
|
||||
LastError = null;
|
||||
log.LogInformation("sync done: {Summary}", LastSummary);
|
||||
return LastSummary;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LastError = e.Message.Length > 300 ? e.Message[..300] : e.Message;
|
||||
log.LogError(e, "sync failed");
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
Syncing = false;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Sync;
|
||||
|
||||
public sealed record SyncOptions(int EmbedChunk = 20, int EmbedMaxChars = EmbeddingText.DefaultMaxChars);
|
||||
|
||||
public sealed record SyncSummary(string Commit, int Changed, int Removed, int Embedded, int Notes, bool Skipped, TimeSpan Elapsed);
|
||||
|
||||
/// <summary>
|
||||
/// Brings the catalog and the active index to the note store's head. Documents whose content
|
||||
/// hash is unchanged are neither read nor re-embedded; wiki documents that changed are embedded
|
||||
/// under every registered embedding model; documents gone from the head are removed.
|
||||
/// A failure leaves the last good state in place.
|
||||
/// </summary>
|
||||
public sealed class SyncUseCase(
|
||||
INoteStore store,
|
||||
IDocumentCatalog catalog,
|
||||
IVectorIndex index,
|
||||
IReadOnlyList<IEmbeddingModel> embeddingModels,
|
||||
TimeProvider clock,
|
||||
ILogger<SyncUseCase> log,
|
||||
SyncOptions? options = null)
|
||||
{
|
||||
private readonly SyncOptions _options = options ?? new SyncOptions();
|
||||
|
||||
private static readonly IReadOnlySet<DocumentKind> AllKinds =
|
||||
new HashSet<DocumentKind> { DocumentKind.Wiki, DocumentKind.Raw, DocumentKind.RawProcessed, DocumentKind.Output };
|
||||
|
||||
private static string Short(string commit) => commit.Length > 8 ? commit[..8] : commit;
|
||||
|
||||
public async Task<SyncSummary> RunAsync(bool force, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(store);
|
||||
var started = clock.GetTimestamp();
|
||||
var head = await store.GetHeadAsync(ct);
|
||||
var state = await catalog.GetStateAsync(ct);
|
||||
var wikiCount = (await catalog.ListAsync(DocumentKind.Wiki, ct)).Count;
|
||||
|
||||
if (!force && state.Commit == head.Commit && wikiCount > 0)
|
||||
{
|
||||
await catalog.SetStateAsync(state with { SyncedAt = clock.GetUtcNow() }, ct);
|
||||
await catalog.FlushAsync(ct);
|
||||
return new SyncSummary(head.Commit, 0, 0, 0, wikiCount, true, clock.GetElapsedTime(started));
|
||||
}
|
||||
|
||||
var listed = await store.ListAsync(head.Commit, AllKinds, ct);
|
||||
var known = new Dictionary<(DocumentKind, string), DocumentRef>();
|
||||
foreach (var kind in AllKinds)
|
||||
{
|
||||
foreach (var r in await catalog.ListAsync(kind, ct))
|
||||
known[(kind, r.Slug)] = r;
|
||||
}
|
||||
|
||||
var changed = listed.Where(r => !known.TryGetValue((r.Kind, r.Slug), out var k) || k.ContentHash != r.ContentHash).ToList();
|
||||
var listedKeys = listed.Select(r => (r.Kind, r.Slug)).ToHashSet();
|
||||
var removed = known.Values.Where(r => !listedKeys.Contains((r.Kind, r.Slug))).ToList();
|
||||
log.LogInformation("sync {Commit}: {Listed} documents, {Changed} changed, {Removed} removed", Short(head.Commit), listed.Count, changed.Count, removed.Count);
|
||||
|
||||
var embedded = 0;
|
||||
foreach (var chunk in changed.Chunk(_options.EmbedChunk))
|
||||
{
|
||||
var documents = new List<Document>(chunk.Length);
|
||||
foreach (var r in chunk)
|
||||
{
|
||||
var text = await store.ReadAsync(r, head.Commit, ct);
|
||||
documents.Add(DocumentParser.Parse(r, text));
|
||||
}
|
||||
await catalog.UpsertAsync(documents, ct);
|
||||
|
||||
var wiki = documents.Where(d => d.Kind == DocumentKind.Wiki).ToList();
|
||||
if (wiki.Count == 0) continue;
|
||||
var texts = wiki.Select(d => EmbeddingText.For(d, _options.EmbedMaxChars)).ToList();
|
||||
foreach (var model in embeddingModels)
|
||||
{
|
||||
var vectors = await model.EmbedAsync(texts, ct);
|
||||
var embeddings = wiki.Select((d, i) => new Embedding(model.ModelId, d.Slug, d.ContentHash, vectors[i])).ToList();
|
||||
await index.UpsertAsync(embeddings, ct);
|
||||
}
|
||||
embedded += wiki.Count;
|
||||
}
|
||||
|
||||
if (removed.Count > 0)
|
||||
{
|
||||
await catalog.RemoveAsync(removed, ct);
|
||||
var removedWiki = removed.Where(r => r.Kind == DocumentKind.Wiki).Select(r => r.Slug).ToList();
|
||||
if (removedWiki.Count > 0)
|
||||
{
|
||||
foreach (var model in embeddingModels)
|
||||
await index.RemoveAsync(model.ModelId, removedWiki, ct);
|
||||
}
|
||||
}
|
||||
|
||||
// Index entries that lag the catalog (a model registered later, an interrupted run) are filled here.
|
||||
foreach (var model in embeddingModels)
|
||||
{
|
||||
var indexState = await index.DescribeAsync(model.ModelId, ct);
|
||||
var wikiDocs = await catalog.ListDocumentsAsync(DocumentKind.Wiki, ct);
|
||||
var missing = wikiDocs.Where(d => !indexState.ContentHashBySlug.TryGetValue(d.Slug, out var h) || h != d.ContentHash).ToList();
|
||||
foreach (var chunk in missing.Chunk(_options.EmbedChunk))
|
||||
{
|
||||
var vectors = await model.EmbedAsync(chunk.Select(d => EmbeddingText.For(d, _options.EmbedMaxChars)).ToList(), ct);
|
||||
await index.UpsertAsync(chunk.Select((d, i) => new Embedding(model.ModelId, d.Slug, d.ContentHash, vectors[i])).ToList(), ct);
|
||||
embedded += chunk.Length;
|
||||
}
|
||||
var stale = indexState.ContentHashBySlug.Keys.Except(wikiDocs.Select(d => d.Slug)).ToList();
|
||||
if (stale.Count > 0)
|
||||
await index.RemoveAsync(model.ModelId, stale, ct);
|
||||
}
|
||||
|
||||
await catalog.SetStateAsync(new CatalogState(head.Commit, clock.GetUtcNow()), ct);
|
||||
await index.FlushAsync(ct);
|
||||
await catalog.FlushAsync(ct);
|
||||
var notes = (await catalog.ListAsync(DocumentKind.Wiki, ct)).Count;
|
||||
return new SyncSummary(head.Commit, changed.Count, removed.Count, embedded, notes, false, clock.GetElapsedTime(started));
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,10 @@
|
||||
<ProjectReference Include="..\Wiki.Core\Wiki.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.12" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace Wiki.Core.Model;
|
||||
|
||||
/// <summary>A paragraph handed to the answer model as evidence, with the judge's probability that it answers.</summary>
|
||||
public sealed record EvidenceParagraph(Paragraph Paragraph, string NoteTitle, double Probability);
|
||||
|
||||
/// <summary>Everything the answer model is allowed to see. Not the whole context: the selected evidence and the judgements.</summary>
|
||||
public sealed record AnswerRequest(
|
||||
string Question,
|
||||
IReadOnlyList<EvidenceParagraph> Evidence,
|
||||
IReadOnlyDictionary<string, Judgement> Judgements,
|
||||
double Covered);
|
||||
|
||||
public abstract record AnswerChunk
|
||||
{
|
||||
/// <summary>A piece of the answer text, in order.</summary>
|
||||
public sealed record Text(string Delta) : AnswerChunk;
|
||||
|
||||
/// <summary>The end of the answer, with what it cost.</summary>
|
||||
public sealed record Done(string ModelId, long InputTokens, long OutputTokens, TimeSpan Elapsed) : AnswerChunk;
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
namespace Wiki.Core.Model;
|
||||
|
||||
/// <summary>Where in the vault a document lives. Kinds are disjoint sets of paths.</summary>
|
||||
public enum DocumentKind
|
||||
{
|
||||
/// <summary>wiki/*.md — the retrieval corpus, embedded.</summary>
|
||||
Wiki,
|
||||
/// <summary>raw/*.md — sources waiting for distillation.</summary>
|
||||
Raw,
|
||||
/// <summary>raw/_processed/*.md — sources distillation has finished with; provenance.</summary>
|
||||
RawProcessed,
|
||||
/// <summary>output/*.md — synthesised answers, audit and consolidation reports.</summary>
|
||||
Output,
|
||||
}
|
||||
|
||||
/// <summary>A listing entry: enough to know whether a document changed without reading it.</summary>
|
||||
public sealed record DocumentRef(DocumentKind Kind, string Slug, string Path, string ContentHash);
|
||||
|
||||
/// <summary>A parsed vault document. Frontmatter fields the vault uses are lifted into properties; the rest stay in <see cref="Frontmatter"/>.</summary>
|
||||
public sealed record Document(
|
||||
DocumentKind Kind,
|
||||
string Slug,
|
||||
string Path,
|
||||
string ContentHash,
|
||||
string Title,
|
||||
string? Type,
|
||||
IReadOnlyList<string> Tags,
|
||||
IReadOnlyList<string> Aliases,
|
||||
IReadOnlyList<string> Related,
|
||||
IReadOnlyList<string> Sources,
|
||||
DateOnly? Updated,
|
||||
IReadOnlyDictionary<string, FrontmatterValue> Frontmatter,
|
||||
string Body,
|
||||
int BodyStartLine)
|
||||
{
|
||||
public DocumentRef Ref => new(Kind, Slug, Path, ContentHash);
|
||||
}
|
||||
|
||||
/// <summary>A frontmatter value: a scalar or a list of scalars. The vault uses nothing deeper.</summary>
|
||||
public sealed record FrontmatterValue(string? Scalar, IReadOnlyList<string>? List)
|
||||
{
|
||||
public static FrontmatterValue Of(string scalar) => new(scalar, null);
|
||||
public static FrontmatterValue Of(IReadOnlyList<string> list) => new(null, list);
|
||||
public bool IsList => List is not null;
|
||||
|
||||
/// <summary>The value as a list: a list as is, a scalar as a one-element list, an empty scalar as empty.</summary>
|
||||
public IReadOnlyList<string> AsList() =>
|
||||
List ?? (string.IsNullOrWhiteSpace(Scalar) ? [] : [Scalar!]);
|
||||
}
|
||||
|
||||
/// <summary>Part of a document body addressable in typed questions: <c>slug#pN</c>.</summary>
|
||||
public sealed record Paragraph(string Id, string Slug, string Heading, string Text);
|
||||
|
||||
/// <summary>An outgoing reference from one document to another, by target slug.</summary>
|
||||
public sealed record Link(string FromSlug, string ToSlug, LinkKind Kind, int? Line);
|
||||
|
||||
public enum LinkKind
|
||||
{
|
||||
/// <summary>[[target]] in the body.</summary>
|
||||
WikiLink,
|
||||
/// <summary>An entry in the <c>related:</c> frontmatter list.</summary>
|
||||
Related,
|
||||
}
|
||||
|
||||
/// <summary>A caller identity for attribution. Resolved by the host's authentication, carried into use cases.</summary>
|
||||
public sealed record Identity(string Name, IdentityKind Kind)
|
||||
{
|
||||
public static readonly Identity Anonymous = new("anonymous", IdentityKind.Anonymous);
|
||||
}
|
||||
|
||||
public enum IdentityKind
|
||||
{
|
||||
Anonymous,
|
||||
Human,
|
||||
Machine,
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
namespace Wiki.Core.Model;
|
||||
|
||||
/// <summary>Result of an adapter's self check. Reported in status, never used for readiness.</summary>
|
||||
public sealed record AdapterHealth(string Adapter, bool Ok, string? Detail, TimeSpan Latency, DateTimeOffset CheckedAt);
|
||||
@@ -0,0 +1,36 @@
|
||||
namespace Wiki.Core.Model;
|
||||
|
||||
/// <summary>
|
||||
/// A typed question over a state. Three primitives, as TypeSafe's System One defines them;
|
||||
/// any judge adapter has to answer all three.
|
||||
/// </summary>
|
||||
public abstract record TypedQuestion(string Instructions)
|
||||
{
|
||||
/// <summary>A yes/no judgement with a probability. Criteria describe both outcomes.</summary>
|
||||
public sealed record Noul(string Instructions, string TrueCriterion, string FalseCriterion) : TypedQuestion(Instructions);
|
||||
|
||||
/// <summary>One option out of many, with a probability per option. Cap is the judge's concern.</summary>
|
||||
public sealed record Choice(string Instructions, IReadOnlyDictionary<string, string> Options) : TypedQuestion(Instructions);
|
||||
|
||||
/// <summary>A graded judgement in [0, 1]. Criteria describe the ends of the scale.</summary>
|
||||
public sealed record Score(string Instructions, string Criteria) : TypedQuestion(Instructions);
|
||||
}
|
||||
|
||||
/// <summary>The state a judge sees plus the questions asked about it, one request.</summary>
|
||||
public sealed record JudgeRequest(object State, IReadOnlyDictionary<string, TypedQuestion> Questions);
|
||||
|
||||
public abstract record Judgement
|
||||
{
|
||||
public sealed record Noul(double Probability) : Judgement;
|
||||
public sealed record Choice(string Selected, IReadOnlyDictionary<string, double> Probabilities, double Confidence) : Judgement;
|
||||
public sealed record Score(double Value) : Judgement;
|
||||
}
|
||||
|
||||
public sealed record JudgeUsage(int Requests, long InputTokens, long OutputTokens, TimeSpan Elapsed)
|
||||
{
|
||||
public static readonly JudgeUsage None = new(0, 0, 0, TimeSpan.Zero);
|
||||
public JudgeUsage Add(JudgeUsage other) =>
|
||||
new(Requests + other.Requests, InputTokens + other.InputTokens, OutputTokens + other.OutputTokens, Elapsed + other.Elapsed);
|
||||
}
|
||||
|
||||
public sealed record JudgeResult(IReadOnlyDictionary<string, Judgement> Answers, JudgeUsage Usage);
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace Wiki.Core.Model;
|
||||
|
||||
/// <summary>
|
||||
/// A fact an agent deposits. Becomes one file in raw/ with provenance frontmatter.
|
||||
/// There is deliberately no field for a hypothesis: the shape enforces the raw rule.
|
||||
/// </summary>
|
||||
public sealed record RawSource(
|
||||
string Title,
|
||||
string Source,
|
||||
string Content,
|
||||
DateTimeOffset ObservedAt,
|
||||
IReadOnlyList<string> Tags)
|
||||
{
|
||||
public const int MinimumContentLength = 40;
|
||||
|
||||
/// <summary>Validation errors, empty when the fact may be stored.</summary>
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
if (string.IsNullOrWhiteSpace(Title)) errors.Add("title is required");
|
||||
if (string.IsNullOrWhiteSpace(Source)) errors.Add("source is required: where the fact was observed or quoted from");
|
||||
if (Content is null || Content.Trim().Length < MinimumContentLength)
|
||||
errors.Add($"content must be at least {MinimumContentLength} characters of observation");
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What the fact sink returns once the source is stored.</summary>
|
||||
public sealed record FactReceipt(string Path, string Commit, DateTimeOffset StoredAt);
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace Wiki.Core.Model;
|
||||
|
||||
/// <summary>A vector for one document under one embedding model.</summary>
|
||||
public sealed record Embedding(string ModelId, string Slug, string ContentHash, float[] Vector);
|
||||
|
||||
/// <summary>One result of a vector query.</summary>
|
||||
public sealed record RankedHit(string Slug, float Score);
|
||||
|
||||
/// <summary>What an index holds for one model: which documents at which content hash.</summary>
|
||||
public sealed record IndexState(string ModelId, IReadOnlyDictionary<string, string> ContentHashBySlug)
|
||||
{
|
||||
public int Count => ContentHashBySlug.Count;
|
||||
}
|
||||
|
||||
/// <summary>Head of the note store: the commit the listing is taken from.</summary>
|
||||
public sealed record StoreHead(string Commit, DateTimeOffset? CommittedAt);
|
||||
@@ -0,0 +1,72 @@
|
||||
using System.Globalization;
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Core.Parsing;
|
||||
|
||||
/// <summary>Turns a vault file into a <see cref="Document"/>. Kind and slug come from the path.</summary>
|
||||
public static class DocumentParser
|
||||
{
|
||||
public static Document Parse(DocumentRef reference, string text)
|
||||
{
|
||||
var (fields, body, bodyStart) = Frontmatter.Parse(text);
|
||||
var title = Field(fields, "title")?.Scalar?.Trim();
|
||||
if (string.IsNullOrEmpty(title)) title = reference.Slug;
|
||||
return new Document(
|
||||
reference.Kind,
|
||||
reference.Slug,
|
||||
reference.Path,
|
||||
reference.ContentHash,
|
||||
title,
|
||||
Field(fields, "type")?.Scalar?.Trim(),
|
||||
ListField(fields, "tags"),
|
||||
ListField(fields, "aliases"),
|
||||
ListField(fields, "related"),
|
||||
ListField(fields, "sources"),
|
||||
ParseDate(Field(fields, "updated")?.Scalar),
|
||||
fields,
|
||||
body,
|
||||
bodyStart);
|
||||
}
|
||||
|
||||
/// <summary>Kind and slug for a repository path, or null when the path is not a vault document.</summary>
|
||||
public static (DocumentKind Kind, string Slug)? Classify(string path)
|
||||
{
|
||||
if (!path.EndsWith(".md", StringComparison.Ordinal)) return null;
|
||||
var slash = path.LastIndexOf('/');
|
||||
var dir = slash < 0 ? string.Empty : path[..(slash + 1)];
|
||||
var name = path[(slash + 1)..^3];
|
||||
if (name.Length == 0 || name.StartsWith('_')) return null; // _index.md and friends
|
||||
return dir switch
|
||||
{
|
||||
"wiki/" => (DocumentKind.Wiki, name),
|
||||
"raw/" => (DocumentKind.Raw, name),
|
||||
"raw/_processed/" => (DocumentKind.RawProcessed, name),
|
||||
"output/" => (DocumentKind.Output, name),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
public static string PathFor(DocumentKind kind, string slug) => kind switch
|
||||
{
|
||||
DocumentKind.Wiki => $"wiki/{slug}.md",
|
||||
DocumentKind.Raw => $"raw/{slug}.md",
|
||||
DocumentKind.RawProcessed => $"raw/_processed/{slug}.md",
|
||||
DocumentKind.Output => $"output/{slug}.md",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(kind)),
|
||||
};
|
||||
|
||||
private static FrontmatterValue? Field(IReadOnlyDictionary<string, FrontmatterValue> fields, string key) =>
|
||||
fields.TryGetValue(key, out var v) ? v : null;
|
||||
|
||||
private static IReadOnlyList<string> ListField(IReadOnlyDictionary<string, FrontmatterValue> fields, string key) =>
|
||||
Field(fields, key)?.AsList() ?? [];
|
||||
|
||||
public static DateOnly? ParseDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value)) return null;
|
||||
var s = value.Trim();
|
||||
if (s.Length >= 10 && DateOnly.TryParseExact(s[..10], "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var d))
|
||||
return d;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// The text a note is embedded from: slug words, title, aliases, tags and the first paragraph
|
||||
/// after the H1, capped. Same composition as the Python wiki-index so the two rank alike.
|
||||
/// </summary>
|
||||
public static class EmbeddingText
|
||||
{
|
||||
public const int DefaultMaxChars = 1500;
|
||||
|
||||
public static string For(Document document, int maxChars = DefaultMaxChars)
|
||||
{
|
||||
var parts = new[]
|
||||
{
|
||||
document.Slug.Replace('-', ' '),
|
||||
document.Title,
|
||||
string.Join(' ', document.Aliases),
|
||||
string.Join(' ', document.Tags),
|
||||
FirstParagraph(document.Body),
|
||||
};
|
||||
var text = string.Join(' ', parts);
|
||||
return text.Length > maxChars ? text[..maxChars] : text;
|
||||
}
|
||||
|
||||
/// <summary>The first non-heading paragraph after the H1; empty when there is no H1.</summary>
|
||||
public static string FirstParagraph(string body)
|
||||
{
|
||||
var para = new List<string>();
|
||||
var seenH1 = false;
|
||||
foreach (var raw in body.Split('\n'))
|
||||
{
|
||||
var line = raw.TrimEnd('\r');
|
||||
if (line.StartsWith("# ", StringComparison.Ordinal))
|
||||
{
|
||||
seenH1 = true;
|
||||
continue;
|
||||
}
|
||||
if (!seenH1) continue;
|
||||
if (line.Trim().Length == 0 || line.StartsWith('#'))
|
||||
{
|
||||
if (para.Count > 0) break;
|
||||
continue;
|
||||
}
|
||||
para.Add(line.Trim());
|
||||
}
|
||||
return string.Join(' ', para);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using System.Text;
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the YAML subset the vault uses: <c>key: scalar</c>, <c>key: [a, "b, c", 'd']</c>,
|
||||
/// block lists of <c>- item</c>, quoted scalars, and nothing nested deeper. Tolerant: a line
|
||||
/// it cannot read is skipped, never fatal, because a note with odd frontmatter is still a note.
|
||||
/// </summary>
|
||||
public static class Frontmatter
|
||||
{
|
||||
public static (IReadOnlyDictionary<string, FrontmatterValue> Fields, string Body, int BodyStartLine) Parse(string text)
|
||||
{
|
||||
var fields = new Dictionary<string, FrontmatterValue>(StringComparer.Ordinal);
|
||||
if (!text.StartsWith("---", StringComparison.Ordinal))
|
||||
return (fields, text, 1);
|
||||
|
||||
var lines = text.Split('\n');
|
||||
if (lines[0].TrimEnd('\r').Trim() != "---")
|
||||
return (fields, text, 1);
|
||||
|
||||
var end = -1;
|
||||
for (var i = 1; i < lines.Length; i++)
|
||||
{
|
||||
if (lines[i].TrimEnd('\r').Trim() == "---") { end = i; break; }
|
||||
}
|
||||
if (end < 0)
|
||||
return (fields, text, 1);
|
||||
|
||||
string? pendingKey = null;
|
||||
List<string>? pendingList = null;
|
||||
for (var i = 1; i < end; i++)
|
||||
{
|
||||
var line = lines[i].TrimEnd('\r');
|
||||
if (string.IsNullOrWhiteSpace(line) || line.TrimStart().StartsWith('#'))
|
||||
continue;
|
||||
|
||||
if (pendingKey is not null && line.TrimStart().StartsWith("- ", StringComparison.Ordinal))
|
||||
{
|
||||
pendingList ??= [];
|
||||
pendingList.Add(Unquote(line.TrimStart()[2..].Trim()));
|
||||
continue;
|
||||
}
|
||||
if (pendingKey is not null)
|
||||
{
|
||||
fields[pendingKey] = pendingList is null ? FrontmatterValue.Of(string.Empty) : FrontmatterValue.Of(pendingList);
|
||||
pendingKey = null;
|
||||
pendingList = null;
|
||||
}
|
||||
|
||||
var colon = IndexOfKeyColon(line);
|
||||
if (colon < 0)
|
||||
continue;
|
||||
var key = line[..colon].Trim();
|
||||
var value = line[(colon + 1)..].Trim();
|
||||
if (key.Length == 0)
|
||||
continue;
|
||||
|
||||
if (value.Length == 0)
|
||||
{
|
||||
pendingKey = key; // a block list may follow; otherwise an empty scalar
|
||||
continue;
|
||||
}
|
||||
if (value.StartsWith('[') && value.EndsWith(']'))
|
||||
{
|
||||
fields[key] = FrontmatterValue.Of(SplitFlowList(value[1..^1]));
|
||||
continue;
|
||||
}
|
||||
fields[key] = FrontmatterValue.Of(Unquote(value));
|
||||
}
|
||||
if (pendingKey is not null)
|
||||
fields[pendingKey] = pendingList is null ? FrontmatterValue.Of(string.Empty) : FrontmatterValue.Of(pendingList);
|
||||
|
||||
var body = string.Join('\n', lines.Skip(end + 1));
|
||||
return (fields, body, end + 2);
|
||||
}
|
||||
|
||||
private static int IndexOfKeyColon(string line)
|
||||
{
|
||||
// A key is [A-Za-z_][\w-]* at column 0 followed by ':' and a space or end of line.
|
||||
var i = 0;
|
||||
if (i >= line.Length || !(char.IsLetter(line[i]) || line[i] == '_')) return -1;
|
||||
while (i < line.Length && (char.IsLetterOrDigit(line[i]) || line[i] == '_' || line[i] == '-')) i++;
|
||||
if (i >= line.Length || line[i] != ':') return -1;
|
||||
if (i + 1 < line.Length && line[i + 1] != ' ' && line[i + 1] != '\t') return -1;
|
||||
return i;
|
||||
}
|
||||
|
||||
/// <summary>Splits <c>a, "b, c", 'd'</c> respecting quotes; sources lists carry commas inside quotes.</summary>
|
||||
public static IReadOnlyList<string> SplitFlowList(string inner)
|
||||
{
|
||||
var items = new List<string>();
|
||||
var current = new StringBuilder();
|
||||
char? quote = null;
|
||||
foreach (var ch in inner)
|
||||
{
|
||||
if (quote is null && (ch == '"' || ch == '\''))
|
||||
{
|
||||
quote = ch;
|
||||
current.Append(ch);
|
||||
continue;
|
||||
}
|
||||
if (quote is not null && ch == quote)
|
||||
{
|
||||
quote = null;
|
||||
current.Append(ch);
|
||||
continue;
|
||||
}
|
||||
if (quote is null && ch == ',')
|
||||
{
|
||||
Push(items, current);
|
||||
continue;
|
||||
}
|
||||
current.Append(ch);
|
||||
}
|
||||
Push(items, current);
|
||||
return items;
|
||||
|
||||
static void Push(List<string> items, StringBuilder sb)
|
||||
{
|
||||
var s = Unquote(sb.ToString().Trim());
|
||||
sb.Clear();
|
||||
if (s.Length > 0) items.Add(s);
|
||||
}
|
||||
}
|
||||
|
||||
public static string Unquote(string value)
|
||||
{
|
||||
if (value.Length >= 2 && ((value[0] == '"' && value[^1] == '"') || (value[0] == '\'' && value[^1] == '\'')))
|
||||
return value[1..^1];
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using System.Text;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Core.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// The vault's paragraph splitter, as wiki_ask.py does it: blank-line separated blocks,
|
||||
/// headings set the block's heading and are not blocks themselves, fenced code is one block.
|
||||
/// Ids are <c>slug#pN</c>, counted from 1 in document order.
|
||||
/// </summary>
|
||||
public sealed class ParagraphChunker : IChunker
|
||||
{
|
||||
public IReadOnlyList<Paragraph> Split(Document document)
|
||||
{
|
||||
var paragraphs = new List<Paragraph>();
|
||||
var heading = document.Title;
|
||||
var block = new StringBuilder();
|
||||
var fenced = false;
|
||||
|
||||
void Flush()
|
||||
{
|
||||
var text = block.ToString().Trim();
|
||||
block.Clear();
|
||||
if (text.Length == 0) return;
|
||||
var n = paragraphs.Count + 1;
|
||||
paragraphs.Add(new Paragraph($"{document.Slug}#p{n}", document.Slug, heading, text));
|
||||
}
|
||||
|
||||
foreach (var raw in document.Body.Split('\n'))
|
||||
{
|
||||
var line = raw.TrimEnd('\r');
|
||||
var trimmed = line.TrimStart();
|
||||
if (trimmed.StartsWith("```", StringComparison.Ordinal) || trimmed.StartsWith("~~~", StringComparison.Ordinal))
|
||||
{
|
||||
if (!fenced) Flush();
|
||||
fenced = !fenced;
|
||||
block.Append(line).Append('\n');
|
||||
if (!fenced) Flush();
|
||||
continue;
|
||||
}
|
||||
if (fenced)
|
||||
{
|
||||
block.Append(line).Append('\n');
|
||||
continue;
|
||||
}
|
||||
if (trimmed.StartsWith('#'))
|
||||
{
|
||||
Flush();
|
||||
heading = trimmed.TrimStart('#').Trim();
|
||||
continue;
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
Flush();
|
||||
continue;
|
||||
}
|
||||
block.Append(line).Append('\n');
|
||||
}
|
||||
Flush();
|
||||
return paragraphs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Core.Parsing;
|
||||
|
||||
/// <summary>Outgoing references of a document: [[links]] in the body outside code, plus the related list.</summary>
|
||||
public static partial class WikiLinks
|
||||
{
|
||||
[GeneratedRegex(@"\[\[([^\]\[|#]+)(?:#[^\]\[|]*)?(?:\|[^\]\[]*)?\]\]")]
|
||||
private static partial Regex LinkPattern();
|
||||
|
||||
[GeneratedRegex(@"^[A-Za-z0-9][A-Za-z0-9._-]*$")]
|
||||
private static partial Regex SlugPattern();
|
||||
|
||||
[GeneratedRegex("`[^`]*`")]
|
||||
private static partial Regex InlineCode();
|
||||
|
||||
public static bool IsSlug(string value) => SlugPattern().IsMatch(value);
|
||||
|
||||
public static IReadOnlyList<Link> Extract(Document document)
|
||||
{
|
||||
var links = new List<Link>();
|
||||
foreach (var target in document.Related)
|
||||
{
|
||||
if (SlugPattern().IsMatch(target))
|
||||
links.Add(new Link(document.Slug, target, LinkKind.Related, null));
|
||||
}
|
||||
|
||||
var fenced = false;
|
||||
var lineNo = document.BodyStartLine - 1;
|
||||
foreach (var raw in document.Body.Split('\n'))
|
||||
{
|
||||
lineNo++;
|
||||
var line = raw.TrimEnd('\r');
|
||||
var trimmed = line.TrimStart();
|
||||
if (trimmed.StartsWith("```", StringComparison.Ordinal) || trimmed.StartsWith("~~~", StringComparison.Ordinal))
|
||||
{
|
||||
fenced = !fenced;
|
||||
continue;
|
||||
}
|
||||
if (fenced) continue;
|
||||
var stripped = InlineCode().Replace(line, string.Empty);
|
||||
foreach (Match m in LinkPattern().Matches(stripped))
|
||||
{
|
||||
var target = m.Groups[1].Value.Trim();
|
||||
if (SlugPattern().IsMatch(target))
|
||||
links.Add(new Link(document.Slug, target, LinkKind.WikiLink, lineNo));
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Core.Ports;
|
||||
|
||||
/// <summary>Anything that can say whether it is reachable and working. Every adapter implements it.</summary>
|
||||
public interface IAdapterHealth
|
||||
{
|
||||
string Name { get; }
|
||||
Task<AdapterHealth> CheckAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Where the vault's documents are read from.</summary>
|
||||
public interface INoteStore : IAdapterHealth
|
||||
{
|
||||
/// <summary>The current head; a listing is always taken at a head.</summary>
|
||||
Task<StoreHead> GetHeadAsync(CancellationToken ct);
|
||||
|
||||
/// <summary>Every document of the given kinds at that head, with a content hash that changes iff the content changes.</summary>
|
||||
Task<IReadOnlyList<DocumentRef>> ListAsync(string commit, IReadOnlySet<DocumentKind> kinds, CancellationToken ct);
|
||||
|
||||
/// <summary>The document's text at that head.</summary>
|
||||
Task<string> ReadAsync(DocumentRef reference, string commit, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Where a deposited fact goes: one new raw source, never an edit of an existing file.</summary>
|
||||
public interface IFactSink : IAdapterHealth
|
||||
{
|
||||
Task<FactReceipt> AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Text to vector. One instance per configured model.</summary>
|
||||
public interface IEmbeddingModel : IAdapterHealth
|
||||
{
|
||||
string ModelId { get; }
|
||||
int Dimension { get; }
|
||||
Task<IReadOnlyList<float[]>> EmbedAsync(IReadOnlyList<string> texts, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Vectors per (model, document) with top-k query. Models are namespaces; nothing is shared across them.</summary>
|
||||
public interface IVectorIndex : IAdapterHealth
|
||||
{
|
||||
Task<IndexState> DescribeAsync(string modelId, CancellationToken ct);
|
||||
Task UpsertAsync(IReadOnlyList<Embedding> embeddings, CancellationToken ct);
|
||||
Task RemoveAsync(string modelId, IReadOnlyList<string> slugs, CancellationToken ct);
|
||||
Task<IReadOnlyList<RankedHit>> QueryAsync(string modelId, float[] query, int k, CancellationToken ct);
|
||||
/// <summary>Persist whatever the adapter keeps in memory. A no-op for adapters that write through.</summary>
|
||||
Task FlushAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>The synced documents, queryable: bodies, frontmatter and the link graph. Filled by the sync, read by everything else.</summary>
|
||||
public interface IDocumentCatalog : IAdapterHealth
|
||||
{
|
||||
Task<CatalogState> GetStateAsync(CancellationToken ct);
|
||||
Task SetStateAsync(CatalogState state, CancellationToken ct);
|
||||
Task UpsertAsync(IReadOnlyList<Document> documents, CancellationToken ct);
|
||||
Task RemoveAsync(IReadOnlyList<DocumentRef> references, CancellationToken ct);
|
||||
Task<Document?> GetAsync(DocumentKind kind, string slug, CancellationToken ct);
|
||||
Task<IReadOnlyList<DocumentRef>> ListAsync(DocumentKind kind, CancellationToken ct);
|
||||
Task<IReadOnlyList<Document>> ListDocumentsAsync(DocumentKind kind, CancellationToken ct);
|
||||
/// <summary>Wiki documents that reference the slug, by wikilink or related entry.</summary>
|
||||
Task<IReadOnlyList<Link>> BacklinksAsync(string slug, CancellationToken ct);
|
||||
Task<IReadOnlyList<Link>> OutgoingLinksAsync(string slug, CancellationToken ct);
|
||||
Task<IReadOnlyDictionary<string, int>> TagCountsAsync(CancellationToken ct);
|
||||
Task FlushAsync(CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Which head the catalog reflects and when it got there.</summary>
|
||||
public sealed record CatalogState(string? Commit, DateTimeOffset? SyncedAt);
|
||||
|
||||
/// <summary>Splits a document body into addressable paragraphs for typed questions.</summary>
|
||||
public interface IChunker
|
||||
{
|
||||
IReadOnlyList<Paragraph> Split(Document document);
|
||||
}
|
||||
|
||||
/// <summary>Typed questions over a state. Returns probabilities, never prose.</summary>
|
||||
public interface IJudge : IAdapterHealth
|
||||
{
|
||||
string ModelId { get; }
|
||||
/// <summary>The largest number of options one Choice may carry, or null for no cap.</summary>
|
||||
int? MaxChoiceOptions { get; }
|
||||
Task<JudgeResult> JudgeAsync(JudgeRequest request, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>Prose from a question and its evidence. Sees only what the request carries.</summary>
|
||||
public interface IAnswerModel : IAdapterHealth
|
||||
{
|
||||
string ModelId { get; }
|
||||
IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, CancellationToken ct);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using Wiki.Application.Status;
|
||||
using Wiki.Application.Sync;
|
||||
|
||||
namespace Wiki.Host.Api;
|
||||
|
||||
/// <summary>/api/v1: the read API grows here phase by phase. Phase 1 exposes status and a manual sync.</summary>
|
||||
public static class ApiV1
|
||||
{
|
||||
public static void MapApiV1(this WebApplication app)
|
||||
{
|
||||
var v1 = app.MapGroup("/api/v1");
|
||||
|
||||
v1.MapGet("/status", async (StatusQuery status, CancellationToken ct) =>
|
||||
Results.Ok(await status.GetAsync(includeAdapterChecks: true, ct))).AllowAnonymous();
|
||||
|
||||
v1.MapPost("/sync", async (SyncCoordinator sync, bool? force, CancellationToken ct) =>
|
||||
{
|
||||
var summary = await sync.RunAsync(force ?? false, ct);
|
||||
return summary is null
|
||||
? Results.Json(new { error = sync.LastError }, statusCode: StatusCodes.Status502BadGateway)
|
||||
: Results.Ok(summary);
|
||||
}).RequireAuthorization(Auth.Policies.Write);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using Wiki.Application.Status;
|
||||
|
||||
namespace Wiki.Host.Api;
|
||||
|
||||
public static class HealthEndpoints
|
||||
{
|
||||
/// <summary>
|
||||
/// /health: the process serves, always 200 (liveness).
|
||||
/// /ready: catalog synced once and the active index loaded (readiness and startup).
|
||||
/// Adapter checks belong to /api/v1/status and never gate readiness.
|
||||
/// </summary>
|
||||
public static void MapHealthEndpoints(this WebApplication app)
|
||||
{
|
||||
app.MapGet("/health", () => Results.Ok(new { ok = true })).AllowAnonymous();
|
||||
app.MapGet("/ready", async (StatusQuery status, CancellationToken ct) =>
|
||||
{
|
||||
var ready = await status.IsReadyAsync(ct);
|
||||
var body = await status.GetAsync(includeAdapterChecks: false, ct);
|
||||
return ready ? Results.Ok(body) : Results.Json(body, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
}).AllowAnonymous();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Wiki.Application.Retrieval;
|
||||
using Wiki.Application.Status;
|
||||
using Wiki.Core.Ports;
|
||||
using Wiki.Host.Composition;
|
||||
|
||||
namespace Wiki.Host.Api;
|
||||
|
||||
public sealed record RankRequest(string? Query, string? Question, int? K, string? Model);
|
||||
|
||||
/// <summary>
|
||||
/// The endpoints the Python wiki-index served, kept shape-compatible so bin/wiki_rank.py works
|
||||
/// unchanged during the migration: POST /rank {query, k} → {rows: [{slug, title, score}], index},
|
||||
/// POST /webhook (Gitea push, HMAC-SHA256 in X-Gitea-Signature).
|
||||
/// </summary>
|
||||
public static class IndexEndpoints
|
||||
{
|
||||
public static void MapIndexEndpoints(this WebApplication app)
|
||||
{
|
||||
app.MapPost("/rank", RankAsync).AllowAnonymous();
|
||||
app.MapPost("/api/v1/rank", RankAsync).AllowAnonymous();
|
||||
|
||||
app.MapPost("/webhook", async (HttpRequest request, IConfiguration configuration, WikiOptions options, SyncBackgroundService sync, CancellationToken ct) =>
|
||||
{
|
||||
var secret = configuration["Wiki:Webhook:Secret"];
|
||||
if (string.IsNullOrEmpty(secret))
|
||||
return Results.Json(new { error = "webhook secret not configured" }, statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
using var ms = new MemoryStream();
|
||||
await request.Body.CopyToAsync(ms, ct);
|
||||
var body = ms.ToArray();
|
||||
var expected = Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(secret), body));
|
||||
var presented = (request.Headers["X-Gitea-Signature"].FirstOrDefault() ?? string.Empty).Trim();
|
||||
if (!CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(expected), Encoding.ASCII.GetBytes(presented)))
|
||||
return Results.Json(new { error = "bad signature" }, statusCode: StatusCodes.Status403Forbidden);
|
||||
|
||||
string? @ref = null;
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(body.Length == 0 ? "{}"u8.ToArray() : body);
|
||||
if (doc.RootElement.TryGetProperty("ref", out var r)) @ref = r.GetString();
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return Results.BadRequest(new { error = "bad json" });
|
||||
}
|
||||
if (!string.IsNullOrEmpty(@ref) && @ref != $"refs/heads/{options.NoteStore.Gitea.Branch}")
|
||||
return Results.Ok(new { queued = false, @ref });
|
||||
sync.Trigger();
|
||||
return Results.Ok(new { queued = true });
|
||||
}).AllowAnonymous();
|
||||
}
|
||||
|
||||
private static async Task<IResult> RankAsync(RankRequest body, RankUseCase rank, StatusQuery status, IServiceProvider services, IEmbeddingModel active, CancellationToken ct)
|
||||
{
|
||||
var question = (body.Query ?? body.Question ?? string.Empty).Trim();
|
||||
if (question.Length == 0) return Results.BadRequest(new { error = "query required" });
|
||||
var model = active;
|
||||
if (!string.IsNullOrEmpty(body.Model))
|
||||
{
|
||||
model = services.GetKeyedService<IEmbeddingModel>(body.Model);
|
||||
if (model is null) return Results.BadRequest(new { error = $"unknown model '{body.Model}'" });
|
||||
}
|
||||
try
|
||||
{
|
||||
var rows = await rank.RankAsync(model, question, body.K ?? 60, ct);
|
||||
var index = await status.GetAsync(includeAdapterChecks: false, ct);
|
||||
return Results.Ok(new { rows = rows.Select(r => new { r.Slug, r.Title, r.Score }), model = model.ModelId, index });
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
return Results.Json(new { error = e.Message }, statusCode: StatusCodes.Status502BadGateway);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Security.Claims;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Encodings.Web;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Wiki.Host.Auth;
|
||||
|
||||
/// <summary>
|
||||
/// Machine callers present `X-Api-Key: <key>` or `Authorization: Bearer <key>`. Keys are named,
|
||||
/// so a write carries the identity behind it: Auth:ApiKeys:{name} = key. Fail closed: no keys
|
||||
/// configured means no key authenticates.
|
||||
/// </summary>
|
||||
public sealed class ApiKeyAuthenticationHandler(
|
||||
IOptionsMonitor<AuthenticationSchemeOptions> options,
|
||||
ILoggerFactory logger,
|
||||
UrlEncoder encoder,
|
||||
IConfiguration configuration)
|
||||
: AuthenticationHandler<AuthenticationSchemeOptions>(options, logger, encoder)
|
||||
{
|
||||
public const string SchemeName = "ApiKey";
|
||||
|
||||
protected override Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||
{
|
||||
var presented = Request.Headers["X-Api-Key"].FirstOrDefault();
|
||||
if (presented is null)
|
||||
{
|
||||
var authorization = Request.Headers.Authorization.FirstOrDefault();
|
||||
if (authorization?.StartsWith("Bearer ", StringComparison.Ordinal) == true)
|
||||
presented = authorization["Bearer ".Length..];
|
||||
}
|
||||
if (presented is null)
|
||||
return Task.FromResult(AuthenticateResult.NoResult());
|
||||
|
||||
var presentedBytes = Encoding.UTF8.GetBytes(presented);
|
||||
foreach (var child in configuration.GetSection("Auth:ApiKeys").GetChildren())
|
||||
{
|
||||
var configured = child.Value;
|
||||
if (string.IsNullOrEmpty(configured)) continue;
|
||||
if (CryptographicOperations.FixedTimeEquals(presentedBytes, Encoding.UTF8.GetBytes(configured)))
|
||||
{
|
||||
var identity = new ClaimsIdentity([new Claim(ClaimTypes.Name, child.Key), new Claim(Claims.Kind, "machine")], Scheme.Name);
|
||||
return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(new ClaimsPrincipal(identity), Scheme.Name)));
|
||||
}
|
||||
}
|
||||
return Task.FromResult(AuthenticateResult.Fail("invalid API key"));
|
||||
}
|
||||
}
|
||||
|
||||
public static class Policies
|
||||
{
|
||||
public const string Write = "write";
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
using System.Net;
|
||||
using System.Security.Claims;
|
||||
using System.Text.Encodings.Web;
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Wiki.Host.Auth;
|
||||
|
||||
/// <summary>
|
||||
/// Humans arrive through the Authentik proxy outpost, which sets X-Authentik-Username after
|
||||
/// validating the session. The header is trusted only from the peers listed in
|
||||
/// Auth:ReverseProxy:TrustedProxies; with none configured the scheme authenticates nobody.
|
||||
/// </summary>
|
||||
public sealed class ProxyHeaderAuthenticationHandler(
|
||||
IOptionsMonitor<AuthenticationSchemeOptions> options,
|
||||
ILoggerFactory logger,
|
||||
UrlEncoder encoder,
|
||||
IConfiguration configuration)
|
||||
: AuthenticationHandler<AuthenticationSchemeOptions>(options, logger, encoder)
|
||||
{
|
||||
public const string SchemeName = "ProxyHeader";
|
||||
public const string DefaultHeaderName = "X-Authentik-Username";
|
||||
|
||||
protected override Task<AuthenticateResult> HandleAuthenticateAsync()
|
||||
{
|
||||
var headerName = configuration["Auth:ReverseProxy:HeaderName"];
|
||||
if (string.IsNullOrWhiteSpace(headerName)) headerName = DefaultHeaderName;
|
||||
|
||||
var user = Request.Headers[headerName].FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(user))
|
||||
return Task.FromResult(AuthenticateResult.NoResult());
|
||||
|
||||
var peer = Context.Connection.RemoteIpAddress;
|
||||
if (peer is null || !IsTrustedProxy(peer))
|
||||
{
|
||||
Logger.LogWarning("ignoring {Header} from untrusted peer {Peer}", headerName, peer);
|
||||
return Task.FromResult(AuthenticateResult.NoResult());
|
||||
}
|
||||
|
||||
var identity = new ClaimsIdentity([new Claim(ClaimTypes.Name, user), new Claim(Claims.Kind, "human")], Scheme.Name);
|
||||
return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(new ClaimsPrincipal(identity), Scheme.Name)));
|
||||
}
|
||||
|
||||
protected override Task HandleChallengeAsync(AuthenticationProperties properties)
|
||||
{
|
||||
Response.StatusCode = StatusCodes.Status401Unauthorized;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private bool IsTrustedProxy(IPAddress peer)
|
||||
{
|
||||
if (peer.IsIPv4MappedToIPv6) peer = peer.MapToIPv4();
|
||||
var trusted = configuration.GetSection("Auth:ReverseProxy:TrustedProxies").Get<string[]>() ?? [];
|
||||
foreach (var entry in trusted)
|
||||
{
|
||||
if (IPAddress.TryParse(entry, out var single))
|
||||
{
|
||||
if (single.Equals(peer)) return true;
|
||||
continue;
|
||||
}
|
||||
if (IPNetwork.TryParse(entry, out var network) && network.Contains(peer))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Claims
|
||||
{
|
||||
public const string Kind = "wiki:kind";
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
<!DOCTYPE html>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
@@ -6,6 +6,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<base href="/" />
|
||||
<ResourcePreloader />
|
||||
<link rel="stylesheet" href="@Assets["_content/MudBlazor/MudBlazor.min.css"]" />
|
||||
<link rel="stylesheet" href="@Assets["app.css"]" />
|
||||
<link rel="stylesheet" href="@Assets["Wiki.Host.styles.css"]" />
|
||||
<ImportMap />
|
||||
@@ -16,6 +17,7 @@
|
||||
<Routes />
|
||||
<ReconnectModal />
|
||||
<script src="@Assets["_framework/blazor.web.js"]"></script>
|
||||
<script src="@Assets["_content/MudBlazor/MudBlazor.min.js"]"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
@@ -1,6 +1,22 @@
|
||||
@inherits LayoutComponentBase
|
||||
@inherits LayoutComponentBase
|
||||
|
||||
@Body
|
||||
<MudThemeProvider />
|
||||
<MudPopoverProvider />
|
||||
<MudSnackbarProvider />
|
||||
<MudDialogProvider />
|
||||
|
||||
<MudLayout>
|
||||
<MudAppBar Elevation="1">
|
||||
<MudText Typo="Typo.h6">Wiki</MudText>
|
||||
<MudSpacer />
|
||||
<MudLink Href="/" Color="Color.Inherit" Class="mx-2">Status</MudLink>
|
||||
</MudAppBar>
|
||||
<MudMainContent>
|
||||
<MudContainer MaxWidth="MaxWidth.Large" Class="py-6">
|
||||
@Body
|
||||
</MudContainer>
|
||||
</MudMainContent>
|
||||
</MudLayout>
|
||||
|
||||
<div id="blazor-error-ui" data-nosnippet>
|
||||
An unhandled error has occurred.
|
||||
|
||||
@@ -1,7 +1,78 @@
|
||||
@page "/"
|
||||
@page "/"
|
||||
@rendermode InteractiveServer
|
||||
@using Wiki.Application.Status
|
||||
@inject StatusQuery Status
|
||||
|
||||
<PageTitle>Home</PageTitle>
|
||||
<PageTitle>Wiki status</PageTitle>
|
||||
|
||||
<h1>Hello, world!</h1>
|
||||
<MudText Typo="Typo.h4" GutterBottom="true">Status</MudText>
|
||||
|
||||
Welcome to your new app.
|
||||
@if (_status is null)
|
||||
{
|
||||
<MudProgressLinear Indeterminate="true" />
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudGrid>
|
||||
<MudItem xs="12" md="6">
|
||||
<MudPaper Class="pa-4">
|
||||
<MudText Typo="Typo.subtitle1">Index</MudText>
|
||||
<MudSimpleTable Dense="true">
|
||||
<tbody>
|
||||
<tr><td>Ready</td><td>@(_status.Ready ? "yes" : "no")</td></tr>
|
||||
<tr><td>Commit</td><td><code>@(_status.Commit is { Length: > 8 } c ? c[..8] : _status.Commit)</code></td></tr>
|
||||
<tr><td>Synced</td><td>@(_status.SyncedAt?.ToString("u") ?? "never") (@(_status.AgeSeconds?.ToString("0") ?? "-") s ago)</td></tr>
|
||||
<tr><td>Syncing</td><td>@(_status.Syncing ? "yes" : "no")</td></tr>
|
||||
<tr><td>Notes</td><td>@_status.Notes</td></tr>
|
||||
<tr><td>Raw queue</td><td>@_status.RawQueue</td></tr>
|
||||
<tr><td>Outputs</td><td>@_status.Outputs</td></tr>
|
||||
@if (_status.LastError is not null)
|
||||
{
|
||||
<tr><td>Last error</td><td><MudText Color="Color.Error">@_status.LastError</MudText></td></tr>
|
||||
}
|
||||
</tbody>
|
||||
</MudSimpleTable>
|
||||
</MudPaper>
|
||||
</MudItem>
|
||||
<MudItem xs="12" md="6">
|
||||
<MudPaper Class="pa-4">
|
||||
<MudText Typo="Typo.subtitle1">Embedding models</MudText>
|
||||
<MudSimpleTable Dense="true">
|
||||
<tbody>
|
||||
@foreach (var i in _status.Indexes)
|
||||
{
|
||||
<tr><td>@i.ModelId @(i.Active ? "(active)" : "")</td><td>@i.Notes vectors</td></tr>
|
||||
}
|
||||
</tbody>
|
||||
</MudSimpleTable>
|
||||
</MudPaper>
|
||||
</MudItem>
|
||||
<MudItem xs="12">
|
||||
<MudPaper Class="pa-4">
|
||||
<MudText Typo="Typo.subtitle1">Adapters</MudText>
|
||||
<MudSimpleTable Dense="true">
|
||||
<tbody>
|
||||
@foreach (var a in _status.Adapters)
|
||||
{
|
||||
<tr>
|
||||
<td>@a.Adapter</td>
|
||||
<td><MudChip T="string" Color="@(a.Ok ? Color.Success : Color.Error)" Size="Size.Small">@(a.Ok ? "ok" : "failing")</MudChip></td>
|
||||
<td>@a.Latency.TotalMilliseconds.ToString("0") ms</td>
|
||||
<td>@a.Detail</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</MudSimpleTable>
|
||||
</MudPaper>
|
||||
</MudItem>
|
||||
</MudGrid>
|
||||
}
|
||||
|
||||
@code {
|
||||
private ServiceStatus? _status;
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
_status = await Status.GetAsync(includeAdapterChecks: true, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
@using System.Net.Http
|
||||
@using System.Net.Http
|
||||
@using System.Net.Http.Json
|
||||
@using Microsoft.AspNetCore.Components.Forms
|
||||
@using Microsoft.AspNetCore.Components.Routing
|
||||
@@ -6,6 +6,7 @@
|
||||
@using static Microsoft.AspNetCore.Components.Web.RenderMode
|
||||
@using Microsoft.AspNetCore.Components.Web.Virtualization
|
||||
@using Microsoft.JSInterop
|
||||
@using MudBlazor
|
||||
@using Wiki.Host
|
||||
@using Wiki.Host.Components
|
||||
@using Wiki.Host.Components.Layout
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
using Wiki.Application.Sync;
|
||||
|
||||
namespace Wiki.Host.Composition;
|
||||
|
||||
/// <summary>Syncs at start, on every webhook trigger, and every interval; retries sooner after a failure.</summary>
|
||||
public sealed class SyncBackgroundService(SyncCoordinator coordinator, WikiOptions options, ILogger<SyncBackgroundService> log) : BackgroundService
|
||||
{
|
||||
private readonly SemaphoreSlim _wake = new(0);
|
||||
|
||||
public void Trigger() => _wake.Release();
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var summary = await coordinator.RunAsync(force: false, stoppingToken);
|
||||
var wait = summary is null ? options.Sync.RetrySeconds : options.Sync.IntervalSeconds;
|
||||
if (summary is null)
|
||||
log.LogWarning("sync failed; retrying in {Seconds}s", wait);
|
||||
try
|
||||
{
|
||||
await _wake.WaitAsync(TimeSpan.FromSeconds(wait), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
// drain extra triggers that arrived during the run; one sync covers them
|
||||
while (_wake.CurrentCount > 0) await _wake.WaitAsync(0, stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
namespace Wiki.Host.Composition;
|
||||
|
||||
/// <summary>The `Wiki` configuration section: which adapters exist and which one is active. Secrets are read separately from configuration keys, never bound here.</summary>
|
||||
public sealed class WikiOptions
|
||||
{
|
||||
public NoteStoreOptions NoteStore { get; set; } = new();
|
||||
public EmbeddingsOptions Embeddings { get; set; } = new();
|
||||
public IndexOptions Index { get; set; } = new();
|
||||
public SyncScheduleOptions Sync { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class NoteStoreOptions
|
||||
{
|
||||
public string Active { get; set; } = "gitea";
|
||||
public GiteaStoreOptions Gitea { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class GiteaStoreOptions
|
||||
{
|
||||
public string BaseUrl { get; set; } = "https://gitea.renner.uno";
|
||||
public string Repository { get; set; } = "k.renner/notes";
|
||||
public string Branch { get; set; } = "main";
|
||||
}
|
||||
|
||||
public sealed class EmbeddingsOptions
|
||||
{
|
||||
public string Active { get; set; } = "bge-m3";
|
||||
public Dictionary<string, EmbeddingModelOptions> Models { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class EmbeddingModelOptions
|
||||
{
|
||||
public string Adapter { get; set; } = "LlamaCpp";
|
||||
public string Url { get; set; } = string.Empty;
|
||||
public string Model { get; set; } = string.Empty;
|
||||
public int Dimension { get; set; } = 1024;
|
||||
}
|
||||
|
||||
public sealed class IndexOptions
|
||||
{
|
||||
public string Active { get; set; } = "file";
|
||||
public FileIndexStoreOptions File { get; set; } = new();
|
||||
}
|
||||
|
||||
public sealed class FileIndexStoreOptions
|
||||
{
|
||||
public string Directory { get; set; } = "/data";
|
||||
}
|
||||
|
||||
public sealed class SyncScheduleOptions
|
||||
{
|
||||
/// <summary>Reconcile interval; the webhook is the fast path.</summary>
|
||||
public int IntervalSeconds { get; set; } = 1800;
|
||||
public int RetrySeconds { get; set; } = 60;
|
||||
}
|
||||
+89
-11
@@ -1,27 +1,105 @@
|
||||
using Microsoft.AspNetCore.Authentication;
|
||||
using MudBlazor.Services;
|
||||
using Wiki.Adapters.FileIndex;
|
||||
using Wiki.Adapters.Gitea;
|
||||
using Wiki.Adapters.LlamaCpp;
|
||||
using Wiki.Application.Retrieval;
|
||||
using Wiki.Application.Status;
|
||||
using Wiki.Application.Sync;
|
||||
using Wiki.Core.Ports;
|
||||
using Wiki.Host.Api;
|
||||
using Wiki.Host.Auth;
|
||||
using Wiki.Host.Components;
|
||||
using Wiki.Host.Composition;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
var wiki = builder.Configuration.GetSection("Wiki").Get<WikiOptions>() ?? new WikiOptions();
|
||||
builder.Services.AddSingleton(wiki);
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
|
||||
// Add services to the container.
|
||||
builder.Services.AddRazorComponents()
|
||||
.AddInteractiveServerComponents();
|
||||
// ---- adapters, selected by configuration ------------------------------------------------
|
||||
builder.Services.AddGiteaAdapters(o =>
|
||||
{
|
||||
o.BaseUrl = wiki.NoteStore.Gitea.BaseUrl;
|
||||
o.Repository = wiki.NoteStore.Gitea.Repository;
|
||||
o.Branch = wiki.NoteStore.Gitea.Branch;
|
||||
o.Token = builder.Configuration["Wiki:NoteStore:Gitea:Token"] ?? string.Empty;
|
||||
});
|
||||
builder.Services.AddFileIndexAdapters(o => o.Directory = wiki.Index.File.Directory);
|
||||
foreach (var (name, model) in wiki.Embeddings.Models)
|
||||
{
|
||||
switch (model.Adapter)
|
||||
{
|
||||
case "LlamaCpp":
|
||||
builder.Services.AddLlamaCppEmbeddings(name, o =>
|
||||
{
|
||||
o.Url = model.Url;
|
||||
o.Model = model.Model;
|
||||
o.ModelId = name;
|
||||
o.Dimension = model.Dimension;
|
||||
o.BasicAuth = builder.Configuration[$"Wiki:Embeddings:Models:{name}:BasicAuth"];
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException($"unknown embeddings adapter '{model.Adapter}' for model '{name}'");
|
||||
}
|
||||
}
|
||||
builder.Services.AddSingleton<IReadOnlyList<IEmbeddingModel>>(sp =>
|
||||
wiki.Embeddings.Models.Keys.Select(sp.GetRequiredKeyedService<IEmbeddingModel>).ToList());
|
||||
builder.Services.AddSingleton<IEmbeddingModel>(sp => sp.GetRequiredKeyedService<IEmbeddingModel>(wiki.Embeddings.Active));
|
||||
builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
|
||||
[
|
||||
sp.GetRequiredService<INoteStore>(),
|
||||
sp.GetRequiredService<IFactSink>(),
|
||||
sp.GetRequiredService<IVectorIndex>(),
|
||||
sp.GetRequiredService<IDocumentCatalog>(),
|
||||
.. sp.GetRequiredService<IReadOnlyList<IEmbeddingModel>>(),
|
||||
]);
|
||||
|
||||
// ---- use cases ---------------------------------------------------------------------------
|
||||
builder.Services.AddSingleton(new SyncOptions());
|
||||
builder.Services.AddSingleton<SyncUseCase>();
|
||||
builder.Services.AddSingleton<SyncCoordinator>();
|
||||
builder.Services.AddSingleton<RankUseCase>();
|
||||
builder.Services.AddSingleton<StatusQuery>();
|
||||
builder.Services.AddSingleton<SyncBackgroundService>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<SyncBackgroundService>());
|
||||
|
||||
// ---- auth: humans arrive through the Authentik outpost, machines with an API key ---------
|
||||
builder.Services.AddAuthentication(ProxyHeaderAuthenticationHandler.SchemeName)
|
||||
.AddScheme<AuthenticationSchemeOptions, ProxyHeaderAuthenticationHandler>(ProxyHeaderAuthenticationHandler.SchemeName, null)
|
||||
.AddScheme<AuthenticationSchemeOptions, ApiKeyAuthenticationHandler>(ApiKeyAuthenticationHandler.SchemeName, null);
|
||||
builder.Services.AddAuthorization(options =>
|
||||
{
|
||||
options.AddPolicy(Policies.Write, p => p.AddAuthenticationSchemes(ApiKeyAuthenticationHandler.SchemeName).RequireAuthenticatedUser());
|
||||
});
|
||||
|
||||
// ---- UI ----------------------------------------------------------------------------------
|
||||
builder.Services.AddRazorComponents().AddInteractiveServerComponents();
|
||||
builder.Services.AddMudServices();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
// Configure the HTTP request pipeline.
|
||||
if (!app.Environment.IsDevelopment())
|
||||
{
|
||||
app.UseExceptionHandler("/Error", createScopeForErrors: true);
|
||||
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
|
||||
app.UseHsts();
|
||||
}
|
||||
app.UseStatusCodePagesWithReExecute("/not-found", createScopeForStatusCodePages: true);
|
||||
app.UseHttpsRedirection();
|
||||
|
||||
// The not-found page is for the UI only: an API 401/404 must stay a JSON status, not be
|
||||
// re-executed as a POST against a Blazor page (which fails antiforgery with an empty 400).
|
||||
app.UseWhen(
|
||||
ctx => HttpMethods.IsGet(ctx.Request.Method) && !ctx.Request.Path.StartsWithSegments("/api"),
|
||||
branch => branch.UseStatusCodePagesWithReExecute("/not-found", createScopeForStatusCodePages: true));
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.UseAntiforgery();
|
||||
|
||||
app.MapHealthEndpoints();
|
||||
app.MapIndexEndpoints();
|
||||
app.MapApiV1();
|
||||
|
||||
app.MapStaticAssets();
|
||||
app.MapRazorComponents<App>()
|
||||
.AddInteractiveServerRenderMode();
|
||||
app.MapRazorComponents<App>().AddInteractiveServerRenderMode();
|
||||
|
||||
app.Run();
|
||||
|
||||
public partial class Program;
|
||||
|
||||
@@ -9,6 +9,10 @@
|
||||
<ProjectReference Include="..\Wiki.Adapters.Anthropic\Wiki.Adapters.Anthropic.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MudBlazor" Version="9.10.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
|
||||
@@ -2,8 +2,27 @@
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Information",
|
||||
"Microsoft.AspNetCore": "Warning"
|
||||
"Microsoft.AspNetCore": "Warning",
|
||||
"System.Net.Http.HttpClient": "Warning"
|
||||
}
|
||||
},
|
||||
"AllowedHosts": "*"
|
||||
"AllowedHosts": "*",
|
||||
"Wiki": {
|
||||
"NoteStore": {
|
||||
"Active": "gitea",
|
||||
"Gitea": { "BaseUrl": "https://gitea.renner.uno", "Repository": "k.renner/notes", "Branch": "main" }
|
||||
},
|
||||
"Embeddings": {
|
||||
"Active": "bge-m3",
|
||||
"Models": {
|
||||
"bge-m3": { "Adapter": "LlamaCpp", "Url": "http://wiki-embed.ai.svc.cluster.local:8080", "Model": "bge-m3", "Dimension": 1024 }
|
||||
}
|
||||
},
|
||||
"Index": { "Active": "file", "File": { "Directory": "/data" } },
|
||||
"Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 }
|
||||
},
|
||||
"Auth": {
|
||||
"ReverseProxy": { "HeaderName": "X-Authentik-Username", "TrustedProxies": [] },
|
||||
"ApiKeys": {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Tests;
|
||||
|
||||
/// <summary>In-memory ports for use-case tests. Content hash = SHA-1 of the text, like a git blob.</summary>
|
||||
public sealed class FakeStore : INoteStore
|
||||
{
|
||||
public Dictionary<string, string> Files { get; } = new();
|
||||
public string Head { get; set; } = "c1";
|
||||
public int Reads { get; private set; }
|
||||
public string Name => "fake-store";
|
||||
|
||||
public Task<StoreHead> GetHeadAsync(CancellationToken ct) => Task.FromResult(new StoreHead(Head, null));
|
||||
|
||||
public Task<IReadOnlyList<DocumentRef>> ListAsync(string commit, IReadOnlySet<DocumentKind> kinds, CancellationToken ct)
|
||||
{
|
||||
var refs = new List<DocumentRef>();
|
||||
foreach (var (path, text) in Files)
|
||||
{
|
||||
var c = DocumentParser.Classify(path);
|
||||
if (c is null || !kinds.Contains(c.Value.Kind)) continue;
|
||||
refs.Add(new DocumentRef(c.Value.Kind, c.Value.Slug, path, Convert.ToHexString(SHA1.HashData(Encoding.UTF8.GetBytes(text)))));
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<DocumentRef>>(refs);
|
||||
}
|
||||
|
||||
public Task<string> ReadAsync(DocumentRef reference, string commit, CancellationToken ct)
|
||||
{
|
||||
Reads++;
|
||||
return Task.FromResult(Files[reference.Path]);
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public sealed class FakeCatalog : IDocumentCatalog
|
||||
{
|
||||
private readonly Dictionary<(DocumentKind, string), Document> _docs = new();
|
||||
private CatalogState _state = new(null, null);
|
||||
public int Flushes { get; private set; }
|
||||
public string Name => "fake-catalog";
|
||||
|
||||
public Task<CatalogState> GetStateAsync(CancellationToken ct) => Task.FromResult(_state);
|
||||
public Task SetStateAsync(CatalogState state, CancellationToken ct) { _state = state; return Task.CompletedTask; }
|
||||
public Task UpsertAsync(IReadOnlyList<Document> documents, CancellationToken ct) { foreach (var d in documents) _docs[(d.Kind, d.Slug)] = d; return Task.CompletedTask; }
|
||||
public Task RemoveAsync(IReadOnlyList<DocumentRef> references, CancellationToken ct) { foreach (var r in references) _docs.Remove((r.Kind, r.Slug)); return Task.CompletedTask; }
|
||||
public Task<Document?> GetAsync(DocumentKind kind, string slug, CancellationToken ct) => Task.FromResult(_docs.GetValueOrDefault((kind, slug)));
|
||||
public Task<IReadOnlyList<DocumentRef>> ListAsync(DocumentKind kind, CancellationToken ct) => Task.FromResult<IReadOnlyList<DocumentRef>>(_docs.Values.Where(d => d.Kind == kind).Select(d => d.Ref).ToList());
|
||||
public Task<IReadOnlyList<Document>> ListDocumentsAsync(DocumentKind kind, CancellationToken ct) => Task.FromResult<IReadOnlyList<Document>>(_docs.Values.Where(d => d.Kind == kind).ToList());
|
||||
public Task<IReadOnlyList<Link>> BacklinksAsync(string slug, CancellationToken ct) => Task.FromResult<IReadOnlyList<Link>>(_docs.Values.Where(d => d.Kind == DocumentKind.Wiki).SelectMany(WikiLinks.Extract).Where(l => l.ToSlug == slug).ToList());
|
||||
public Task<IReadOnlyList<Link>> OutgoingLinksAsync(string slug, CancellationToken ct) => Task.FromResult<IReadOnlyList<Link>>(_docs.TryGetValue((DocumentKind.Wiki, slug), out var d) ? WikiLinks.Extract(d) : []);
|
||||
public Task<IReadOnlyDictionary<string, int>> TagCountsAsync(CancellationToken ct) => Task.FromResult<IReadOnlyDictionary<string, int>>(_docs.Values.Where(d => d.Kind == DocumentKind.Wiki).SelectMany(d => d.Tags).GroupBy(t => t).ToDictionary(g => g.Key, g => g.Count()));
|
||||
public Task FlushAsync(CancellationToken ct) { Flushes++; return Task.CompletedTask; }
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public sealed class FakeIndex : IVectorIndex
|
||||
{
|
||||
public Dictionary<string, Dictionary<string, (string Hash, float[] Vector)>> Models { get; } = new();
|
||||
public int Flushes { get; private set; }
|
||||
public string Name => "fake-index";
|
||||
|
||||
private Dictionary<string, (string Hash, float[] Vector)> Of(string model) => Models.TryGetValue(model, out var m) ? m : Models[model] = new();
|
||||
|
||||
public Task<IndexState> DescribeAsync(string modelId, CancellationToken ct) => Task.FromResult(new IndexState(modelId, Of(modelId).ToDictionary(kv => kv.Key, kv => kv.Value.Hash)));
|
||||
public Task UpsertAsync(IReadOnlyList<Embedding> embeddings, CancellationToken ct) { foreach (var e in embeddings) Of(e.ModelId)[e.Slug] = (e.ContentHash, e.Vector); return Task.CompletedTask; }
|
||||
public Task RemoveAsync(string modelId, IReadOnlyList<string> slugs, CancellationToken ct) { foreach (var s in slugs) Of(modelId).Remove(s); return Task.CompletedTask; }
|
||||
public Task<IReadOnlyList<RankedHit>> QueryAsync(string modelId, float[] query, int k, CancellationToken ct) =>
|
||||
Task.FromResult<IReadOnlyList<RankedHit>>(Of(modelId).Select(kv => new RankedHit(kv.Key, kv.Value.Vector.Zip(query, (a, b) => a * b).Sum())).OrderByDescending(h => h.Score).Take(k).ToList());
|
||||
public Task FlushAsync(CancellationToken ct) { Flushes++; return Task.CompletedTask; }
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
/// <summary>Deterministic 4-dimensional "embeddings": a hash of the text spread over the axes, unit length.</summary>
|
||||
public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel
|
||||
{
|
||||
public string ModelId { get; } = modelId;
|
||||
public int Dimension => 4;
|
||||
public List<IReadOnlyList<string>> Batches { get; } = [];
|
||||
public string Name => "fake-embeddings:" + ModelId;
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EmbedAsync(IReadOnlyList<string> texts, CancellationToken ct)
|
||||
{
|
||||
Batches.Add(texts);
|
||||
var vectors = texts.Select(t =>
|
||||
{
|
||||
var h = SHA256.HashData(Encoding.UTF8.GetBytes(ModelId + t));
|
||||
var v = new float[] { h[0] + 1, h[1] + 1, h[2] + 1, h[3] + 1 };
|
||||
var n = MathF.Sqrt(v.Sum(x => x * x));
|
||||
return v.Select(x => x / n).ToArray();
|
||||
}).ToList();
|
||||
return Task.FromResult<IReadOnlyList<float[]>>(vectors);
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Wiki.Application.Retrieval;
|
||||
using Wiki.Application.Sync;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Application.Tests;
|
||||
|
||||
public class SyncUseCaseTests
|
||||
{
|
||||
private static string Note(string title, string body = "Body.", string tags = "[a]") =>
|
||||
$"---\ntitle: {title}\ntype: wiki\ntags: {tags}\nsources: [x]\nupdated: 2026-09-01\n---\n\n# {title}\n\n{body}\n";
|
||||
|
||||
private readonly FakeStore _store = new();
|
||||
private readonly FakeCatalog _catalog = new();
|
||||
private readonly FakeIndex _index = new();
|
||||
private readonly FakeEmbeddings _model = new("m1");
|
||||
|
||||
private SyncUseCase Sync(params IEmbeddingModel[] models) =>
|
||||
new(_store, _catalog, _index, models.Length == 0 ? [_model] : models, TimeProvider.System, NullLogger<SyncUseCase>.Instance, new SyncOptions(EmbedChunk: 2));
|
||||
|
||||
[Fact]
|
||||
public async Task InitialSyncEmbedsWikiOnlyAndCatalogsEverything()
|
||||
{
|
||||
_store.Files["wiki/a.md"] = Note("A");
|
||||
_store.Files["wiki/b.md"] = Note("B");
|
||||
_store.Files["wiki/c.md"] = Note("C");
|
||||
_store.Files["raw/2026-09-23-r.md"] = "---\ntitle: R\n---\n\nraw\n";
|
||||
_store.Files["output/2026-09-23-o.md"] = "---\ntitle: O\n---\n\nout\n";
|
||||
_store.Files["wiki/_index.md"] = "index";
|
||||
|
||||
var summary = await Sync().RunAsync(false, CancellationToken.None);
|
||||
|
||||
Assert.False(summary.Skipped);
|
||||
Assert.Equal(5, summary.Changed);
|
||||
Assert.Equal(3, summary.Embedded);
|
||||
Assert.Equal(3, summary.Notes);
|
||||
Assert.Equal(3, (await _index.DescribeAsync("m1", CancellationToken.None)).Count);
|
||||
Assert.Single(await _catalog.ListAsync(DocumentKind.Raw, CancellationToken.None));
|
||||
Assert.Single(await _catalog.ListAsync(DocumentKind.Output, CancellationToken.None));
|
||||
Assert.Equal("c1", (await _catalog.GetStateAsync(CancellationToken.None)).Commit);
|
||||
Assert.All(_model.Batches, b => Assert.InRange(b.Count, 1, 2));
|
||||
Assert.Equal(1, _index.Flushes);
|
||||
Assert.Equal(1, _catalog.Flushes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnchangedHeadIsSkippedWithoutReads()
|
||||
{
|
||||
_store.Files["wiki/a.md"] = Note("A");
|
||||
await Sync().RunAsync(false, CancellationToken.None);
|
||||
var reads = _store.Reads;
|
||||
|
||||
var summary = await Sync().RunAsync(false, CancellationToken.None);
|
||||
|
||||
Assert.True(summary.Skipped);
|
||||
Assert.Equal(reads, _store.Reads);
|
||||
Assert.NotNull((await _catalog.GetStateAsync(CancellationToken.None)).SyncedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ChangedDocumentIsReembeddedAndRemovedOneDropped()
|
||||
{
|
||||
_store.Files["wiki/a.md"] = Note("A");
|
||||
_store.Files["wiki/b.md"] = Note("B");
|
||||
await Sync().RunAsync(false, CancellationToken.None);
|
||||
var before = (await _index.DescribeAsync("m1", CancellationToken.None)).ContentHashBySlug;
|
||||
|
||||
_store.Head = "c2";
|
||||
_store.Files["wiki/a.md"] = Note("A", "Changed body.");
|
||||
_store.Files.Remove("wiki/b.md");
|
||||
var reads = _store.Reads;
|
||||
|
||||
var summary = await Sync().RunAsync(false, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, summary.Changed);
|
||||
Assert.Equal(1, summary.Removed);
|
||||
Assert.Equal(1, _store.Reads - reads);
|
||||
var after = (await _index.DescribeAsync("m1", CancellationToken.None)).ContentHashBySlug;
|
||||
Assert.Equal(["a"], after.Keys);
|
||||
Assert.NotEqual(before["a"], after["a"]);
|
||||
Assert.Null(await _catalog.GetAsync(DocumentKind.Wiki, "b", CancellationToken.None));
|
||||
Assert.Contains("Changed body", (await _catalog.GetAsync(DocumentKind.Wiki, "a", CancellationToken.None))!.Body);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ForceResyncsAnUnchangedHead()
|
||||
{
|
||||
_store.Files["wiki/a.md"] = Note("A");
|
||||
await Sync().RunAsync(false, CancellationToken.None);
|
||||
var summary = await Sync().RunAsync(true, CancellationToken.None);
|
||||
Assert.False(summary.Skipped);
|
||||
Assert.Equal(0, summary.Changed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ALaterRegisteredModelIsBackfilledFromTheCatalog()
|
||||
{
|
||||
_store.Files["wiki/a.md"] = Note("A");
|
||||
_store.Files["wiki/b.md"] = Note("B");
|
||||
await Sync().RunAsync(false, CancellationToken.None);
|
||||
|
||||
var second = new FakeEmbeddings("m2");
|
||||
var summary = await Sync(_model, second).RunAsync(true, CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, summary.Embedded);
|
||||
Assert.Equal(2, (await _index.DescribeAsync("m2", CancellationToken.None)).Count);
|
||||
Assert.Equal(2, (await _index.DescribeAsync("m1", CancellationToken.None)).Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RankReturnsTitlesFromTheCatalog()
|
||||
{
|
||||
_store.Files["wiki/udm-dns.md"] = Note("UDM intercepts DNS", "The UDM answers DNS.");
|
||||
_store.Files["wiki/ceph-osd.md"] = Note("Ceph OSD", "OSD fsid.");
|
||||
await Sync().RunAsync(false, CancellationToken.None);
|
||||
|
||||
var rank = new RankUseCase(_index, _catalog);
|
||||
var udm = (await _catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!;
|
||||
var rows = await rank.RankAsync(_model, Wiki.Core.Parsing.EmbeddingText.For(udm), 5, CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, rows.Count);
|
||||
Assert.Equal("udm-dns", rows[0].Slug);
|
||||
Assert.Equal("UDM intercepts DNS", rows[0].Title);
|
||||
Assert.InRange(rows[0].Score, 0.99f, 1.01f);
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => rank.RankAsync(_model, " ", 5, CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CoordinatorRecordsFailureAndRecovers()
|
||||
{
|
||||
_store.Files["wiki/a.md"] = Note("A");
|
||||
var coordinator = new SyncCoordinator(Sync(), TimeProvider.System, NullLogger<SyncCoordinator>.Instance);
|
||||
_store.Head = null!; // GetHeadAsync returns a null commit → the use case fails on Substring
|
||||
Assert.Null(await coordinator.RunAsync(false, CancellationToken.None));
|
||||
Assert.NotNull(coordinator.LastError);
|
||||
Assert.False(coordinator.Syncing);
|
||||
|
||||
_store.Head = "c1";
|
||||
Assert.NotNull(await coordinator.RunAsync(false, CancellationToken.None));
|
||||
Assert.Null(coordinator.LastError);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts;
|
||||
|
||||
/// <summary>Every <see cref="IDocumentCatalog"/> adapter derives from this.</summary>
|
||||
public abstract class DocumentCatalogContract
|
||||
{
|
||||
protected abstract Task<IDocumentCatalog> CreateCatalogAsync();
|
||||
protected abstract Task<IDocumentCatalog> ReopenAsync(IDocumentCatalog catalog);
|
||||
|
||||
private static Document Doc(string path, string hash = "h") =>
|
||||
DocumentParser.Parse(Ref(path, hash), VaultFixture.Files[path]);
|
||||
|
||||
private static DocumentRef Ref(string path, string hash = "h")
|
||||
{
|
||||
var (kind, slug) = DocumentParser.Classify(path)!.Value;
|
||||
return new DocumentRef(kind, slug, path, hash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StartsEmptyWithNoState()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
var state = await catalog.GetStateAsync(CancellationToken.None);
|
||||
Assert.Null(state.Commit);
|
||||
Assert.Empty(await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertGetListByKind()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md"), Doc("raw/2026-09-23-pending.md"), Doc("output/2026-09-20-answer.md")], CancellationToken.None);
|
||||
|
||||
var got = await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None);
|
||||
Assert.NotNull(got);
|
||||
Assert.Equal("UDM intercepts DNS", got.Title);
|
||||
Assert.Contains("answers every UDP/53", got.Body);
|
||||
Assert.Null(await catalog.GetAsync(DocumentKind.Raw, "udm-dns", CancellationToken.None));
|
||||
|
||||
Assert.Equal(2, (await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None)).Count);
|
||||
Assert.Single(await catalog.ListAsync(DocumentKind.Raw, CancellationToken.None));
|
||||
Assert.Single(await catalog.ListDocumentsAsync(DocumentKind.Output, CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LinksAndBacklinksFollowWikilinksAndRelated()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None);
|
||||
var outgoing = await catalog.OutgoingLinksAsync("udm-dns", CancellationToken.None);
|
||||
Assert.Equal([LinkKind.Related, LinkKind.WikiLink], outgoing.Select(l => l.Kind));
|
||||
Assert.All(outgoing, l => Assert.Equal("ceph-osd", l.ToSlug));
|
||||
var back = await catalog.BacklinksAsync("ceph-osd", CancellationToken.None);
|
||||
Assert.Equal(2, back.Count);
|
||||
Assert.All(back, l => Assert.Equal("udm-dns", l.FromSlug));
|
||||
Assert.Empty(await catalog.BacklinksAsync("udm-dns", CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveDropsDocumentAndItsLinks()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None);
|
||||
await catalog.RemoveAsync([Ref("wiki/udm-dns.md")], CancellationToken.None);
|
||||
Assert.Null(await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None));
|
||||
Assert.Empty(await catalog.BacklinksAsync("ceph-osd", CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertReplacesLinks()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
await catalog.UpsertAsync([Doc("wiki/udm-dns.md")], CancellationToken.None);
|
||||
var changed = Doc("wiki/udm-dns.md", "h2") with { Body = "no links any more", Related = [] };
|
||||
await catalog.UpsertAsync([changed], CancellationToken.None);
|
||||
Assert.Empty(await catalog.OutgoingLinksAsync("udm-dns", CancellationToken.None));
|
||||
Assert.Equal("h2", (await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!.ContentHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TagCountsCoverWikiOnly()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md"), Doc("output/2026-09-20-answer.md")], CancellationToken.None);
|
||||
var tags = await catalog.TagCountsAsync(CancellationToken.None);
|
||||
Assert.Equal(1, tags["dns"]);
|
||||
Assert.Equal(1, tags["udm"]);
|
||||
Assert.Equal(1, tags["ceph"]);
|
||||
Assert.Equal(3, tags.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StateAndDocumentsSurviveFlushAndReopen()
|
||||
{
|
||||
var catalog = await CreateCatalogAsync();
|
||||
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None);
|
||||
var at = new DateTimeOffset(2026, 9, 24, 10, 0, 0, TimeSpan.Zero);
|
||||
await catalog.SetStateAsync(new CatalogState(VaultFixture.Commit, at), CancellationToken.None);
|
||||
await catalog.FlushAsync(CancellationToken.None);
|
||||
|
||||
var reopened = await ReopenAsync(catalog);
|
||||
var state = await reopened.GetStateAsync(CancellationToken.None);
|
||||
Assert.Equal(VaultFixture.Commit, state.Commit);
|
||||
Assert.Equal(at, state.SyncedAt);
|
||||
Assert.Equal(2, (await reopened.ListAsync(DocumentKind.Wiki, CancellationToken.None)).Count);
|
||||
Assert.Equal(2, (await reopened.BacklinksAsync("ceph-osd", CancellationToken.None)).Count);
|
||||
Assert.Equal("UDM intercepts DNS", (await reopened.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!.Title);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts;
|
||||
|
||||
/// <summary>Every <see cref="IEmbeddingModel"/> adapter derives from this. The adapter under test may talk to a fake server.</summary>
|
||||
public abstract class EmbeddingModelContract
|
||||
{
|
||||
protected abstract Task<IEmbeddingModel> CreateModelAsync();
|
||||
|
||||
[Fact]
|
||||
public async Task VectorsHaveTheDeclaredDimensionAndUnitLength()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
var vectors = await model.EmbedAsync(["alpha", "beta"], CancellationToken.None);
|
||||
Assert.Equal(2, vectors.Count);
|
||||
foreach (var v in vectors)
|
||||
{
|
||||
Assert.Equal(model.Dimension, v.Length);
|
||||
var norm = MathF.Sqrt(v.Sum(x => x * x));
|
||||
Assert.InRange(norm, 0.99f, 1.01f);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SameTextSameVectorAndOrderPreserved()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
var once = await model.EmbedAsync(["one", "two", "three"], CancellationToken.None);
|
||||
var again = await model.EmbedAsync(["three", "one"], CancellationToken.None);
|
||||
Assert.Equal(once[2], again[0]);
|
||||
Assert.Equal(once[0], again[1]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyBatchIsEmpty()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
Assert.Empty(await model.EmbedAsync([], CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ModelIdIsSet()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
Assert.False(string.IsNullOrWhiteSpace(model.ModelId));
|
||||
var health = await model.CheckAsync(CancellationToken.None);
|
||||
Assert.True(health.Ok, health.Detail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Adapters.FileIndex;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.FileIndex;
|
||||
|
||||
public sealed class FileDocumentCatalogContractTests : DocumentCatalogContract, IDisposable
|
||||
{
|
||||
private readonly List<string> _directories = [];
|
||||
private readonly Dictionary<IDocumentCatalog, string> _directoryOf = new(ReferenceEqualityComparer.Instance);
|
||||
|
||||
protected override Task<IDocumentCatalog> CreateCatalogAsync()
|
||||
{
|
||||
var directory = NewDirectory();
|
||||
IDocumentCatalog catalog = new Wiki.Adapters.FileIndex.FileDocumentCatalog(Options.Create(new FileIndexOptions { Directory = directory }));
|
||||
_directoryOf[catalog] = directory;
|
||||
return Task.FromResult(catalog);
|
||||
}
|
||||
|
||||
protected override Task<IDocumentCatalog> ReopenAsync(IDocumentCatalog catalog)
|
||||
{
|
||||
var directory = _directoryOf[catalog];
|
||||
IDocumentCatalog reopened = new Wiki.Adapters.FileIndex.FileDocumentCatalog(Options.Create(new FileIndexOptions { Directory = directory }));
|
||||
_directoryOf[reopened] = directory;
|
||||
return Task.FromResult(reopened);
|
||||
}
|
||||
|
||||
private string NewDirectory()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "wiki-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
_directories.Add(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var directory in _directories)
|
||||
{
|
||||
try { Directory.Delete(directory, recursive: true); }
|
||||
catch (IOException) { }
|
||||
catch (UnauthorizedAccessException) { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Adapters.FileIndex;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.FileIndex;
|
||||
|
||||
/// <summary>Behaviour specific to the file-backed adapters that the shared ports contracts don't exercise: atomicity and resilience to corrupt state on disk.</summary>
|
||||
public sealed class FileIndexUnitTests : IDisposable
|
||||
{
|
||||
private readonly List<string> _directories = [];
|
||||
|
||||
private string NewDirectory()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "wiki-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
_directories.Add(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
private static IOptions<FileIndexOptions> OptionsFor(string directory) => Options.Create(new FileIndexOptions { Directory = directory });
|
||||
|
||||
private static float[] Unit(params float[] v)
|
||||
{
|
||||
var n = MathF.Sqrt(v.Sum(x => x * x));
|
||||
return v.Select(x => x / n).ToArray();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task VectorIndexFlushLeavesNoTempFileBehind()
|
||||
{
|
||||
var directory = NewDirectory();
|
||||
var index = new FileVectorIndex(OptionsFor(directory));
|
||||
await index.UpsertAsync([new Embedding("m", "a", "h1", Unit(1, 0, 0))], CancellationToken.None);
|
||||
await index.FlushAsync(CancellationToken.None);
|
||||
|
||||
var indexDirectory = Path.Combine(directory, "index");
|
||||
Assert.Empty(Directory.GetFiles(indexDirectory, "*.tmp"));
|
||||
Assert.NotEmpty(Directory.GetFiles(indexDirectory, "*.bin"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task VectorIndexCorruptFileStartsEmptyAndHealthMentionsReset()
|
||||
{
|
||||
var directory = NewDirectory();
|
||||
var indexDirectory = Path.Combine(directory, "index");
|
||||
Directory.CreateDirectory(indexDirectory);
|
||||
await File.WriteAllBytesAsync(Path.Combine(indexDirectory, "broken.bin"), [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]);
|
||||
|
||||
var index = new FileVectorIndex(OptionsFor(directory));
|
||||
var state = await index.DescribeAsync("test-model", CancellationToken.None);
|
||||
Assert.Equal(0, state.Count);
|
||||
|
||||
var health = await index.CheckAsync(CancellationToken.None);
|
||||
Assert.True(health.Ok);
|
||||
Assert.NotNull(health.Detail);
|
||||
Assert.Contains("reset", health.Detail, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CatalogFlushLeavesNoTempFileBehind()
|
||||
{
|
||||
var directory = NewDirectory();
|
||||
var catalog = new FileDocumentCatalog(OptionsFor(directory));
|
||||
await catalog.SetStateAsync(new CatalogState("abc", DateTimeOffset.UtcNow), CancellationToken.None);
|
||||
await catalog.FlushAsync(CancellationToken.None);
|
||||
|
||||
Assert.Empty(Directory.GetFiles(directory, "*.tmp"));
|
||||
Assert.True(File.Exists(Path.Combine(directory, "catalog.json")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CatalogCorruptFileStartsEmptyAndHealthMentionsReset()
|
||||
{
|
||||
var directory = NewDirectory();
|
||||
await File.WriteAllTextAsync(Path.Combine(directory, "catalog.json"), "{ not valid json");
|
||||
|
||||
var catalog = new FileDocumentCatalog(OptionsFor(directory));
|
||||
var state = await catalog.GetStateAsync(CancellationToken.None);
|
||||
Assert.Null(state.Commit);
|
||||
Assert.Empty(await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None));
|
||||
|
||||
var health = await catalog.CheckAsync(CancellationToken.None);
|
||||
Assert.True(health.Ok);
|
||||
Assert.NotNull(health.Detail);
|
||||
Assert.Contains("reset", health.Detail, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var directory in _directories)
|
||||
{
|
||||
try { Directory.Delete(directory, recursive: true); }
|
||||
catch (IOException) { }
|
||||
catch (UnauthorizedAccessException) { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Adapters.FileIndex;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.FileIndex;
|
||||
|
||||
public sealed class FileVectorIndexContractTests : VectorIndexContract, IDisposable
|
||||
{
|
||||
private readonly List<string> _directories = [];
|
||||
private readonly Dictionary<IVectorIndex, string> _directoryOf = new(ReferenceEqualityComparer.Instance);
|
||||
|
||||
protected override Task<IVectorIndex> CreateIndexAsync()
|
||||
{
|
||||
var directory = NewDirectory();
|
||||
IVectorIndex index = new Wiki.Adapters.FileIndex.FileVectorIndex(Options.Create(new FileIndexOptions { Directory = directory }));
|
||||
_directoryOf[index] = directory;
|
||||
return Task.FromResult(index);
|
||||
}
|
||||
|
||||
protected override Task<IVectorIndex> ReopenAsync(IVectorIndex index)
|
||||
{
|
||||
var directory = _directoryOf[index];
|
||||
IVectorIndex reopened = new Wiki.Adapters.FileIndex.FileVectorIndex(Options.Create(new FileIndexOptions { Directory = directory }));
|
||||
_directoryOf[reopened] = directory;
|
||||
return Task.FromResult(reopened);
|
||||
}
|
||||
|
||||
private string NewDirectory()
|
||||
{
|
||||
var directory = Path.Combine(Path.GetTempPath(), "wiki-tests", Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(directory);
|
||||
_directories.Add(directory);
|
||||
return directory;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var directory in _directories)
|
||||
{
|
||||
try { Directory.Delete(directory, recursive: true); }
|
||||
catch (IOException) { }
|
||||
catch (UnauthorizedAccessException) { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
|
||||
namespace Wiki.Contracts.Gitea;
|
||||
|
||||
/// <summary>
|
||||
/// A minimal Gitea API double, in front of an in-memory copy of <see cref="VaultFixture.Files"/>.
|
||||
/// Reproduces the quirks the real Gitea note-store adapter has to tolerate: the tree listing pages
|
||||
/// with "tree": null past the last page, and may report truncated=true even on a page that turns
|
||||
/// out to be the last one with data. Blob shas are a SHA-1 hex of the file's content, so they are
|
||||
/// stable across repeated calls and distinct per distinct content, exactly what the note-store
|
||||
/// contract requires.
|
||||
/// </summary>
|
||||
public sealed class FakeGiteaHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly Dictionary<string, string> _files;
|
||||
private readonly string _repo;
|
||||
private readonly string _branch;
|
||||
private readonly string _commit;
|
||||
private readonly int _pageSize;
|
||||
|
||||
/// <summary>Every file created via POST .../contents/{path}, path -> decoded text, in call order.</summary>
|
||||
public List<(string Path, string Content, string Branch, string Message)> Created { get; } = [];
|
||||
|
||||
public FakeGiteaHandler(string repository = "k.renner/notes", string branch = "main", int pageSize = 3)
|
||||
{
|
||||
_repo = repository;
|
||||
_branch = branch;
|
||||
_commit = VaultFixture.Commit;
|
||||
_pageSize = pageSize;
|
||||
_files = new Dictionary<string, string>(VaultFixture.Files);
|
||||
}
|
||||
|
||||
/// <summary>Adds a file to the fake as if it already existed at HEAD, e.g. to force the
|
||||
/// fact sink into its -2 suffix path.</summary>
|
||||
public void SeedFile(string path, string content) => _files[path] = content;
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
var path = request.RequestUri!.AbsolutePath;
|
||||
var prefix = $"/api/v1/repos/{_repo}/";
|
||||
if (!path.StartsWith(prefix, StringComparison.Ordinal))
|
||||
return Task.FromResult(NotFound());
|
||||
|
||||
var rest = path[prefix.Length..];
|
||||
|
||||
if (request.Method == HttpMethod.Get && rest == $"branches/{_branch}")
|
||||
return Task.FromResult(Json(new JsonObject
|
||||
{
|
||||
["commit"] = new JsonObject
|
||||
{
|
||||
["id"] = _commit,
|
||||
["timestamp"] = "2026-09-20T12:00:00Z",
|
||||
},
|
||||
}));
|
||||
|
||||
if (request.Method == HttpMethod.Get && rest.StartsWith($"git/trees/{_commit}", StringComparison.Ordinal))
|
||||
return Task.FromResult(HandleTree(request.RequestUri));
|
||||
|
||||
if (request.Method == HttpMethod.Get && rest.StartsWith("raw/", StringComparison.Ordinal))
|
||||
return Task.FromResult(HandleRaw(rest["raw/".Length..]));
|
||||
|
||||
if (request.Method == HttpMethod.Get && rest.StartsWith("contents/", StringComparison.Ordinal))
|
||||
return Task.FromResult(HandleContentsExists(rest["contents/".Length..]));
|
||||
|
||||
if (request.Method == HttpMethod.Post && rest.StartsWith("contents/", StringComparison.Ordinal))
|
||||
return HandleContentsCreateAsync(rest["contents/".Length..], request, ct);
|
||||
|
||||
return Task.FromResult(NotFound());
|
||||
}
|
||||
|
||||
private HttpResponseMessage HandleTree(Uri requestUri)
|
||||
{
|
||||
var page = QueryInt(requestUri, "page", 1);
|
||||
var entries = BuildEntries();
|
||||
var start = (page - 1) * _pageSize;
|
||||
|
||||
var body = new JsonObject
|
||||
{
|
||||
["sha"] = _commit,
|
||||
["total_count"] = entries.Count,
|
||||
};
|
||||
|
||||
if (start >= entries.Count)
|
||||
{
|
||||
// Gitea answers "tree": null once you page past the end.
|
||||
body["tree"] = null;
|
||||
return Json(body);
|
||||
}
|
||||
|
||||
var slice = entries.Skip(start).Take(_pageSize).ToList();
|
||||
var tree = new JsonArray();
|
||||
foreach (var e in slice)
|
||||
{
|
||||
tree.Add(new JsonObject
|
||||
{
|
||||
["path"] = e.Path,
|
||||
["type"] = e.Type,
|
||||
["sha"] = e.Sha,
|
||||
});
|
||||
}
|
||||
body["tree"] = tree;
|
||||
// Gitea sets this even when this page happens to be the last one with data; callers must
|
||||
// not use it to decide whether to stop.
|
||||
body["truncated"] = true;
|
||||
return Json(body);
|
||||
}
|
||||
|
||||
private HttpResponseMessage HandleRaw(string encodedPathAndQuery)
|
||||
{
|
||||
var filePath = StripQuery(encodedPathAndQuery);
|
||||
return _files.TryGetValue(filePath, out var content)
|
||||
? new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(content, Encoding.UTF8, "text/plain") }
|
||||
: NotFound();
|
||||
}
|
||||
|
||||
private HttpResponseMessage HandleContentsExists(string encodedPathAndQuery)
|
||||
{
|
||||
var filePath = StripQuery(encodedPathAndQuery);
|
||||
return _files.ContainsKey(filePath)
|
||||
? Json(new JsonObject { ["path"] = filePath })
|
||||
: NotFound();
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> HandleContentsCreateAsync(string encodedPathAndQuery, HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
var filePath = StripQuery(encodedPathAndQuery);
|
||||
var payload = await request.Content!.ReadFromJsonAsync<JsonObject>(ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException("empty create-file request body");
|
||||
var base64 = payload["content"]!.GetValue<string>();
|
||||
var branch = payload["branch"]?.GetValue<string>() ?? _branch;
|
||||
var message = payload["message"]?.GetValue<string>() ?? "";
|
||||
var text = Encoding.UTF8.GetString(Convert.FromBase64String(base64));
|
||||
|
||||
if (_files.ContainsKey(filePath))
|
||||
return NotFound(HttpStatusCode.UnprocessableEntity, $"{filePath} already exists");
|
||||
|
||||
_files[filePath] = text;
|
||||
Created.Add((filePath, text, branch, message));
|
||||
|
||||
return Json(new JsonObject
|
||||
{
|
||||
["commit"] = new JsonObject { ["sha"] = Sha1Hex(text) },
|
||||
});
|
||||
}
|
||||
|
||||
private static string StripQuery(string encodedPathAndQuery)
|
||||
{
|
||||
var raw = encodedPathAndQuery.Contains('?') ? encodedPathAndQuery[..encodedPathAndQuery.IndexOf('?')] : encodedPathAndQuery;
|
||||
return Uri.UnescapeDataString(raw);
|
||||
}
|
||||
|
||||
private static int QueryInt(Uri uri, string key, int fallback)
|
||||
{
|
||||
var query = uri.Query.TrimStart('?');
|
||||
foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var kv = pair.Split('=', 2);
|
||||
if (kv.Length == 2 && kv[0] == key && int.TryParse(Uri.UnescapeDataString(kv[1]), out var value))
|
||||
return value;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
private List<(string Path, string Type, string Sha)> BuildEntries()
|
||||
{
|
||||
var dirs = new SortedSet<string>(StringComparer.Ordinal);
|
||||
foreach (var filePath in _files.Keys)
|
||||
{
|
||||
var slash = filePath.LastIndexOf('/');
|
||||
while (slash > 0)
|
||||
{
|
||||
dirs.Add(filePath[..slash]);
|
||||
slash = filePath.LastIndexOf('/', slash - 1);
|
||||
}
|
||||
}
|
||||
|
||||
var entries = new List<(string Path, string Type, string Sha)>();
|
||||
foreach (var dir in dirs)
|
||||
entries.Add((dir, "tree", Sha1Hex($"dir:{dir}")));
|
||||
foreach (var (filePath, content) in _files)
|
||||
entries.Add((filePath, "blob", Sha1Hex(content)));
|
||||
|
||||
return entries.OrderBy(e => e.Path, StringComparer.Ordinal).ToList();
|
||||
}
|
||||
|
||||
private static string Sha1Hex(string content)
|
||||
{
|
||||
var bytes = SHA1.HashData(Encoding.UTF8.GetBytes(content));
|
||||
return Convert.ToHexStringLower(bytes);
|
||||
}
|
||||
|
||||
private static HttpResponseMessage Json(JsonNode? body) =>
|
||||
new(HttpStatusCode.OK) { Content = new StringContent(body?.ToJsonString() ?? "null", Encoding.UTF8, "application/json") };
|
||||
|
||||
private static HttpResponseMessage NotFound(HttpStatusCode status = HttpStatusCode.NotFound, string message = "not found") =>
|
||||
new(status) { Content = new StringContent(JsonSerializer.Serialize(new { message }), Encoding.UTF8, "application/json") };
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Adapters.Gitea;
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Contracts.Gitea;
|
||||
|
||||
public sealed class GiteaFactSinkTests
|
||||
{
|
||||
private const string Repository = "k.renner/notes";
|
||||
private const string Branch = "main";
|
||||
|
||||
private static readonly RawSource Fact = new(
|
||||
Title: "UDM intercepts DNS: a follow-up",
|
||||
Source: "https://example.com/thread#5",
|
||||
Content: "The UDM answers every UDP/53 query itself, independent of what upstream is configured elsewhere.",
|
||||
ObservedAt: new DateTimeOffset(2026, 9, 20, 14, 30, 0, TimeSpan.Zero),
|
||||
Tags: ["dns", "udm"]);
|
||||
|
||||
private static readonly Identity By = new("agent-x", IdentityKind.Machine);
|
||||
private static readonly DateTimeOffset Now = new(2026, 9, 23, 8, 0, 0, TimeSpan.Zero);
|
||||
|
||||
private const string ExpectedPath = "raw/2026-09-23-udm-intercepts-dns-a-follow-up.md";
|
||||
|
||||
private static readonly string ExpectedText =
|
||||
"---\n" +
|
||||
"title: \"UDM intercepts DNS: a follow-up\"\n" +
|
||||
"type: raw\n" +
|
||||
"source: \"https://example.com/thread#5\"\n" +
|
||||
"captured: 2026-09-23\n" +
|
||||
"captured_by: agent-x\n" +
|
||||
"observed: 2026-09-20T14:30:00+00:00\n" +
|
||||
"tags: [dns, udm]\n" +
|
||||
"---\n" +
|
||||
"\n" +
|
||||
"The UDM answers every UDP/53 query itself, independent of what upstream is configured elsewhere.\n";
|
||||
|
||||
private static (FakeGiteaHandler Handler, GiteaFactSink Sink) CreateSink()
|
||||
{
|
||||
var handler = new FakeGiteaHandler(Repository, Branch, pageSize: 3);
|
||||
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://gitea.test") };
|
||||
var options = Options.Create(new GiteaOptions
|
||||
{
|
||||
BaseUrl = "https://gitea.test",
|
||||
Repository = Repository,
|
||||
Branch = Branch,
|
||||
Token = "test-token",
|
||||
});
|
||||
return (handler, new GiteaFactSink(httpClient, options));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WritesTheExpectedPathAndFrontmatter()
|
||||
{
|
||||
var (handler, sink) = CreateSink();
|
||||
|
||||
var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None);
|
||||
|
||||
Assert.Equal(ExpectedPath, receipt.Path);
|
||||
Assert.Equal(Now, receipt.StoredAt);
|
||||
|
||||
var created = Assert.Single(handler.Created);
|
||||
Assert.Equal(ExpectedPath, created.Path);
|
||||
Assert.Equal(ExpectedText, created.Content);
|
||||
Assert.Equal(Branch, created.Branch);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReturnsTheCommitShaFromTheResponse()
|
||||
{
|
||||
var (_, sink) = CreateSink();
|
||||
|
||||
var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None);
|
||||
|
||||
Assert.Equal(Sha1Hex(ExpectedText), receipt.Commit);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppendsANumberedSuffixWhenThePathAlreadyExists()
|
||||
{
|
||||
var (handler, sink) = CreateSink();
|
||||
handler.SeedFile(ExpectedPath, "already here");
|
||||
|
||||
var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None);
|
||||
|
||||
Assert.Equal("raw/2026-09-23-udm-intercepts-dns-a-follow-up-2.md", receipt.Path);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task KeepsNumberingWhileCandidatesCollide()
|
||||
{
|
||||
var (handler, sink) = CreateSink();
|
||||
handler.SeedFile(ExpectedPath, "already here");
|
||||
handler.SeedFile("raw/2026-09-23-udm-intercepts-dns-a-follow-up-2.md", "also here");
|
||||
|
||||
var receipt = await sink.AppendAsync(Fact, By, Now, CancellationToken.None);
|
||||
|
||||
Assert.Equal("raw/2026-09-23-udm-intercepts-dns-a-follow-up-3.md", receipt.Path);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("", "source", "content long enough to pass the minimum length check, easily.")]
|
||||
[InlineData("title", "", "content long enough to pass the minimum length check, easily.")]
|
||||
[InlineData("title", "source", "too short")]
|
||||
public async Task RejectsAnInvalidRawSourceBeforeCallingGitea(string title, string source, string content)
|
||||
{
|
||||
var (handler, sink) = CreateSink();
|
||||
var invalid = new RawSource(title, source, content, Now, []);
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => sink.AppendAsync(invalid, By, Now, CancellationToken.None));
|
||||
|
||||
Assert.Empty(handler.Created);
|
||||
}
|
||||
|
||||
private static string Sha1Hex(string content)
|
||||
{
|
||||
var bytes = SHA1.HashData(Encoding.UTF8.GetBytes(content));
|
||||
return Convert.ToHexStringLower(bytes);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using Wiki.Adapters.Gitea;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.Gitea;
|
||||
|
||||
public sealed class GiteaNoteStoreContractTests : NoteStoreContract
|
||||
{
|
||||
private const string Repository = "k.renner/notes";
|
||||
private const string Branch = "main";
|
||||
|
||||
protected override Task<INoteStore> CreateStoreAsync()
|
||||
{
|
||||
var handler = new FakeGiteaHandler(Repository, Branch, pageSize: 3);
|
||||
var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://gitea.test") };
|
||||
var options = Options.Create(new GiteaOptions
|
||||
{
|
||||
BaseUrl = "https://gitea.test",
|
||||
Repository = Repository,
|
||||
Branch = Branch,
|
||||
Token = "test-token",
|
||||
});
|
||||
return Task.FromResult<INoteStore>(new GiteaNoteStore(httpClient, options));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Net;
|
||||
using Wiki.Adapters.Gitea;
|
||||
|
||||
namespace Wiki.Contracts.Gitea;
|
||||
|
||||
public class TransientRetryHandlerTests
|
||||
{
|
||||
private sealed class Script(params Func<HttpResponseMessage>[] steps) : HttpMessageHandler
|
||||
{
|
||||
public int Calls { get; private set; }
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
var step = steps[Math.Min(Calls, steps.Length - 1)];
|
||||
Calls++;
|
||||
return Task.FromResult(step());
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpClient Client(Script script, int attempts = 3) =>
|
||||
new(new TransientRetryHandler(attempts, TimeSpan.FromMilliseconds(1), (_, _) => Task.CompletedTask) { InnerHandler = script });
|
||||
|
||||
[Fact]
|
||||
public async Task RetriesAGetAfterAConnectionResetAndA502()
|
||||
{
|
||||
var script = new Script(
|
||||
() => throw new HttpRequestException("reset", new IOException("Connection reset by peer")),
|
||||
() => new HttpResponseMessage(HttpStatusCode.BadGateway),
|
||||
() => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("ok") });
|
||||
var response = await Client(script).GetAsync("http://gitea/x");
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Equal(3, script.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GivesUpAfterTheConfiguredAttempts()
|
||||
{
|
||||
var script = new Script(() => throw new HttpRequestException("reset"));
|
||||
await Assert.ThrowsAsync<HttpRequestException>(() => Client(script, 3).GetAsync("http://gitea/x"));
|
||||
Assert.Equal(3, script.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DoesNotRetryAPostOrA404()
|
||||
{
|
||||
var post = new Script(() => throw new HttpRequestException("reset"));
|
||||
await Assert.ThrowsAsync<HttpRequestException>(() => Client(post).PostAsync("http://gitea/x", new StringContent("{}")));
|
||||
Assert.Equal(1, post.Calls);
|
||||
|
||||
var notFound = new Script(() => new HttpResponseMessage(HttpStatusCode.NotFound));
|
||||
var response = await Client(notFound).GetAsync("http://gitea/x");
|
||||
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
|
||||
Assert.Equal(1, notFound.Calls);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Wiki.Contracts.LlamaCpp;
|
||||
|
||||
/// <summary>
|
||||
/// Stands in for a llama-server. Serves GET /health (togglable) and POST /v1/embeddings,
|
||||
/// producing a deterministic, NOT normalised vector per input text and returning rows in a
|
||||
/// shuffled order tagged with their true "index" so callers must sort by it.
|
||||
/// </summary>
|
||||
internal sealed class FakeLlamaCppHandler : HttpMessageHandler
|
||||
{
|
||||
public List<int> BatchSizesSeen { get; } = [];
|
||||
|
||||
public bool HealthEndpointAvailable { get; set; } = true;
|
||||
|
||||
public int Dimension { get; init; } = 8;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = request.RequestUri!.AbsolutePath;
|
||||
|
||||
if (request.Method == HttpMethod.Get && path.EndsWith("/health", StringComparison.Ordinal))
|
||||
{
|
||||
return HealthEndpointAvailable
|
||||
? new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"status\":\"ok\"}") }
|
||||
: new HttpResponseMessage(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
if (request.Method == HttpMethod.Post && path.EndsWith("/v1/embeddings", StringComparison.Ordinal))
|
||||
{
|
||||
var body = await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
var texts = doc.RootElement.GetProperty("input").EnumerateArray().Select(e => e.GetString()!).ToArray();
|
||||
BatchSizesSeen.Add(texts.Length);
|
||||
|
||||
var rows = texts
|
||||
.Select((text, index) => new { index, embedding = FakeVector(text) })
|
||||
.Reverse() // shuffled: exercises sorting by "index" in the adapter
|
||||
.ToArray();
|
||||
|
||||
var json = JsonSerializer.Serialize(new { data = rows });
|
||||
return new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Unexpected request: {request.Method} {request.RequestUri}");
|
||||
}
|
||||
|
||||
private float[] FakeVector(string text)
|
||||
{
|
||||
var rng = new Random(StableHash(text));
|
||||
var vector = new float[Dimension];
|
||||
for (var i = 0; i < Dimension; i++)
|
||||
{
|
||||
// Deliberately not unit length: 1..2 range, never zero, so the adapter's own
|
||||
// normalisation is what makes the contract's unit-length assertion pass.
|
||||
vector[i] = 1f + rng.NextSingle();
|
||||
}
|
||||
|
||||
return vector;
|
||||
}
|
||||
|
||||
/// <summary>FNV-1a over UTF-8 bytes: stable across runs, unlike string.GetHashCode().</summary>
|
||||
private static int StableHash(string s)
|
||||
{
|
||||
unchecked
|
||||
{
|
||||
var hash = 2166136261;
|
||||
foreach (var b in Encoding.UTF8.GetBytes(s))
|
||||
{
|
||||
hash = (hash ^ b) * 16777619;
|
||||
}
|
||||
|
||||
return (int)hash;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
using Wiki.Adapters.LlamaCpp;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.LlamaCpp;
|
||||
|
||||
/// <summary>Runs the shared <see cref="EmbeddingModelContract"/> against the llama.cpp adapter over a fake server.</summary>
|
||||
public sealed class LlamaCppEmbeddingContractTests : EmbeddingModelContract
|
||||
{
|
||||
private readonly FakeLlamaCppHandler _handler = new() { Dimension = 8 };
|
||||
|
||||
protected override Task<IEmbeddingModel> CreateModelAsync()
|
||||
{
|
||||
var client = new HttpClient(_handler) { BaseAddress = new Uri("http://llama.test/") };
|
||||
var options = new LlamaCppEmbeddingOptions
|
||||
{
|
||||
Url = "http://llama.test/",
|
||||
Model = "bge-m3",
|
||||
Dimension = 8,
|
||||
BatchSize = 2,
|
||||
};
|
||||
|
||||
IEmbeddingModel model = new LlamaCppEmbeddingModel(client, options);
|
||||
return Task.FromResult(model);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RequestsAreBatchedByTheConfiguredSize()
|
||||
{
|
||||
var model = await CreateModelAsync();
|
||||
|
||||
await model.EmbedAsync(["a", "b", "c", "d", "e"], CancellationToken.None);
|
||||
|
||||
Assert.Equal([2, 2, 1], _handler.BatchSizesSeen);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
using Wiki.Adapters.LlamaCpp;
|
||||
|
||||
namespace Wiki.Contracts.LlamaCpp;
|
||||
|
||||
public sealed class LlamaCppEmbeddingModelTests
|
||||
{
|
||||
private static LlamaCppEmbeddingModel CreateModel(FakeLlamaCppHandler handler, int dimension = 8)
|
||||
{
|
||||
var client = new HttpClient(handler) { BaseAddress = new Uri("http://llama.test/") };
|
||||
var options = new LlamaCppEmbeddingOptions
|
||||
{
|
||||
Url = "http://llama.test/",
|
||||
Model = "bge-m3",
|
||||
Dimension = dimension,
|
||||
};
|
||||
|
||||
return new LlamaCppEmbeddingModel(client, options);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DimensionMismatchThrowsNamingActualAndExpected()
|
||||
{
|
||||
var handler = new FakeLlamaCppHandler { Dimension = 4 };
|
||||
var model = CreateModel(handler, dimension: 8);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => model.EmbedAsync(["hello"], CancellationToken.None));
|
||||
|
||||
Assert.Contains("4", ex.Message);
|
||||
Assert.Contains("8", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckAsyncUsesHealthEndpointWithoutEmbedding()
|
||||
{
|
||||
var handler = new FakeLlamaCppHandler { HealthEndpointAvailable = true };
|
||||
var model = CreateModel(handler);
|
||||
|
||||
var health = await model.CheckAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(health.Ok, health.Detail);
|
||||
Assert.Empty(handler.BatchSizesSeen);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CheckAsyncFallsBackToEmbeddingWhenHealthIsMissing()
|
||||
{
|
||||
var handler = new FakeLlamaCppHandler { HealthEndpointAvailable = false };
|
||||
var model = CreateModel(handler);
|
||||
|
||||
var health = await model.CheckAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(health.Ok, health.Detail);
|
||||
Assert.Equal([1], handler.BatchSizesSeen);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Wiki.Adapters.LlamaCpp;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts.LlamaCpp;
|
||||
|
||||
public sealed class LlamaCppServiceCollectionExtensionsTests
|
||||
{
|
||||
[Fact]
|
||||
public void RegistersAKeyedEmbeddingModel()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLlamaCppEmbeddings("embed", o =>
|
||||
{
|
||||
o.Url = "http://wiki-embed.ai.svc.cluster.local:8080";
|
||||
o.Model = "bge-m3";
|
||||
o.Dimension = 1024;
|
||||
});
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
var model = provider.GetRequiredKeyedService<IEmbeddingModel>("embed");
|
||||
|
||||
Assert.Equal("bge-m3", model.ModelId);
|
||||
Assert.Equal(1024, model.Dimension);
|
||||
Assert.Equal("llamacpp-embeddings:bge-m3", model.Name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BasicAuthHeaderIsSentWhenConfigured()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLlamaCppEmbeddings("embed", o =>
|
||||
{
|
||||
o.Url = "http://wiki-embed.example/";
|
||||
o.Model = "bge-m3";
|
||||
o.Dimension = 8;
|
||||
o.BasicAuth = "user:pass";
|
||||
});
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
var factory = provider.GetRequiredService<IHttpClientFactory>();
|
||||
var client = factory.CreateClient("llamacpp-embeddings:embed");
|
||||
|
||||
Assert.NotNull(client.DefaultRequestHeaders.Authorization);
|
||||
Assert.Equal("Basic", client.DefaultRequestHeaders.Authorization!.Scheme);
|
||||
Assert.Equal(
|
||||
Convert.ToBase64String(Encoding.UTF8.GetBytes("user:pass")),
|
||||
client.DefaultRequestHeaders.Authorization.Parameter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NoAuthorizationHeaderWhenBasicAuthNotConfigured()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLlamaCppEmbeddings("embed", o =>
|
||||
{
|
||||
o.Url = "http://wiki-embed.example/";
|
||||
o.Model = "bge-m3";
|
||||
o.Dimension = 8;
|
||||
});
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
var factory = provider.GetRequiredService<IHttpClientFactory>();
|
||||
var client = factory.CreateClient("llamacpp-embeddings:embed");
|
||||
|
||||
Assert.Null(client.DefaultRequestHeaders.Authorization);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("", "bge-m3", 8)]
|
||||
[InlineData("http://x/", "", 8)]
|
||||
[InlineData("http://x/", "bge-m3", 0)]
|
||||
public void InvalidOptionsAreRejectedEagerly(string url, string model, int dimension)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
services.AddLlamaCppEmbeddings("embed", o =>
|
||||
{
|
||||
o.Url = url;
|
||||
o.Model = model;
|
||||
o.Dimension = dimension;
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts;
|
||||
|
||||
/// <summary>Every <see cref="INoteStore"/> adapter derives a test class from this and supplies a store over <see cref="VaultFixture"/>.</summary>
|
||||
public abstract class NoteStoreContract
|
||||
{
|
||||
/// <summary>A store whose current head exposes exactly <see cref="VaultFixture.Files"/>.</summary>
|
||||
protected abstract Task<INoteStore> CreateStoreAsync();
|
||||
|
||||
[Fact]
|
||||
public async Task HeadHasACommit()
|
||||
{
|
||||
var store = await CreateStoreAsync();
|
||||
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||
Assert.False(string.IsNullOrWhiteSpace(head.Commit));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ListsOnlyVaultDocumentsOfRequestedKinds()
|
||||
{
|
||||
var store = await CreateStoreAsync();
|
||||
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||
var all = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||
Assert.Equal(
|
||||
VaultFixture.Expected.OrderBy(e => e.Path),
|
||||
all.Select(r => (r.Kind, r.Slug, r.Path)).OrderBy(e => e.Path));
|
||||
|
||||
var wiki = await store.ListAsync(head.Commit, new HashSet<DocumentKind> { DocumentKind.Wiki }, CancellationToken.None);
|
||||
Assert.All(wiki, r => Assert.Equal(DocumentKind.Wiki, r.Kind));
|
||||
Assert.Equal(2, wiki.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ContentHashIsStableAndDistinct()
|
||||
{
|
||||
var store = await CreateStoreAsync();
|
||||
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||
var a = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||
var b = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||
Assert.Equal(a.OrderBy(r => r.Path).Select(r => r.ContentHash), b.OrderBy(r => r.Path).Select(r => r.ContentHash));
|
||||
Assert.All(a, r => Assert.False(string.IsNullOrWhiteSpace(r.ContentHash)));
|
||||
Assert.Equal(a.Count, a.Select(r => r.ContentHash).Distinct().Count());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadsExactText()
|
||||
{
|
||||
var store = await CreateStoreAsync();
|
||||
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||
var refs = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||
foreach (var r in refs)
|
||||
{
|
||||
var text = await store.ReadAsync(r, head.Commit, CancellationToken.None);
|
||||
Assert.Equal(VaultFixture.Files[r.Path], text);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthReportsOk()
|
||||
{
|
||||
var store = await CreateStoreAsync();
|
||||
var health = await store.CheckAsync(CancellationToken.None);
|
||||
Assert.True(health.Ok, health.Detail);
|
||||
Assert.Equal(store.Name, health.Adapter);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using Wiki.Core.Model;
|
||||
|
||||
namespace Wiki.Contracts;
|
||||
|
||||
/// <summary>
|
||||
/// A tiny vault every note-store adapter has to expose identically. Adapters under test are
|
||||
/// pointed at a fake or a temp copy holding exactly these files; the contract checks that the
|
||||
/// listing and reads come back as written.
|
||||
/// </summary>
|
||||
public static class VaultFixture
|
||||
{
|
||||
public const string Commit = "0123456789abcdef0123456789abcdef01234567";
|
||||
|
||||
public static readonly IReadOnlyDictionary<string, string> Files = new Dictionary<string, string>
|
||||
{
|
||||
["wiki/_index.md"] = "---\ntitle: index\ntype: index\n---\n",
|
||||
["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns, udm]\nsources: [raw/_processed/2026-09-01-dns.md]\nrelated: [ceph-osd]\nupdated: 2026-09-16\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every UDP/53 query itself. See [[ceph-osd]].\n",
|
||||
["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\ntags: [ceph]\nsources: [raw/_processed/2026-09-01-dns.md]\nupdated: 2026-09-10\n---\n\n# Ceph OSD\n\nOSD fsid mismatch after rebuild.\n",
|
||||
["raw/2026-09-23-pending.md"] = "---\ntitle: Pending source\ntype: raw\nsource: test\ncaptured: 2026-09-23\n---\n\nVerbatim capture.\n",
|
||||
["raw/_processed/2026-09-01-dns.md"] = "---\ntitle: Processed source\ntype: raw\nsource: test\ncaptured: 2026-09-01\n---\n\nVerbatim capture, processed.\n",
|
||||
["output/2026-09-20-answer.md"] = "---\ntitle: An answer\ntype: output\nquestion: Does the UDM intercept DNS?\nsources: [udm-dns]\n---\n\nYes.\n",
|
||||
["bin/wiki_rank.py"] = "#!/usr/bin/env python3\n",
|
||||
["Untitled.md"] = "not a vault document\n",
|
||||
};
|
||||
|
||||
/// <summary>The refs a listing over all kinds must return (order not significant).</summary>
|
||||
public static readonly IReadOnlyList<(DocumentKind Kind, string Slug, string Path)> Expected =
|
||||
[
|
||||
(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md"),
|
||||
(DocumentKind.Wiki, "ceph-osd", "wiki/ceph-osd.md"),
|
||||
(DocumentKind.Raw, "2026-09-23-pending", "raw/2026-09-23-pending.md"),
|
||||
(DocumentKind.RawProcessed, "2026-09-01-dns", "raw/_processed/2026-09-01-dns.md"),
|
||||
(DocumentKind.Output, "2026-09-20-answer", "output/2026-09-20-answer.md"),
|
||||
];
|
||||
|
||||
public static readonly IReadOnlySet<DocumentKind> AllKinds =
|
||||
new HashSet<DocumentKind> { DocumentKind.Wiki, DocumentKind.Raw, DocumentKind.RawProcessed, DocumentKind.Output };
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Contracts;
|
||||
|
||||
/// <summary>Every <see cref="IVectorIndex"/> adapter derives from this. Vectors are unit length; scores are cosine.</summary>
|
||||
public abstract class VectorIndexContract
|
||||
{
|
||||
/// <summary>A fresh, empty index. Called again after <see cref="ReopenAsync"/> semantics are needed.</summary>
|
||||
protected abstract Task<IVectorIndex> CreateIndexAsync();
|
||||
|
||||
/// <summary>The same index reopened from persisted state, or the same instance for adapters that keep no files.</summary>
|
||||
protected abstract Task<IVectorIndex> ReopenAsync(IVectorIndex index);
|
||||
|
||||
private const string Model = "test-model";
|
||||
|
||||
private static float[] Unit(params float[] v)
|
||||
{
|
||||
var n = MathF.Sqrt(v.Sum(x => x * x));
|
||||
return v.Select(x => x / n).ToArray();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyIndexDescribesEmptyAndQueriesNothing()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
var state = await index.DescribeAsync(Model, CancellationToken.None);
|
||||
Assert.Equal(0, state.Count);
|
||||
Assert.Empty(await index.QueryAsync(Model, Unit(1, 0, 0), 5, CancellationToken.None));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task QueryRanksByCosineAndHonoursK()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
await index.UpsertAsync(
|
||||
[
|
||||
new Embedding(Model, "a", "h1", Unit(1, 0, 0)),
|
||||
new Embedding(Model, "b", "h2", Unit(0.9f, 0.1f, 0)),
|
||||
new Embedding(Model, "c", "h3", Unit(0, 1, 0)),
|
||||
new Embedding(Model, "d", "h4", Unit(0, 0, 1)),
|
||||
], CancellationToken.None);
|
||||
|
||||
var hits = await index.QueryAsync(Model, Unit(1, 0, 0), 2, CancellationToken.None);
|
||||
Assert.Equal(["a", "b"], hits.Select(h => h.Slug));
|
||||
Assert.True(hits[0].Score > hits[1].Score);
|
||||
Assert.InRange(hits[0].Score, 0.999f, 1.001f);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertReplacesAndDescribeTracksHashes()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0))], CancellationToken.None);
|
||||
await index.UpsertAsync([new Embedding(Model, "a", "h2", Unit(0, 1, 0))], CancellationToken.None);
|
||||
var state = await index.DescribeAsync(Model, CancellationToken.None);
|
||||
Assert.Equal(1, state.Count);
|
||||
Assert.Equal("h2", state.ContentHashBySlug["a"]);
|
||||
var hits = await index.QueryAsync(Model, Unit(0, 1, 0), 1, CancellationToken.None);
|
||||
Assert.Equal("a", hits[0].Slug);
|
||||
Assert.InRange(hits[0].Score, 0.999f, 1.001f);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RemoveDropsSlugs()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0)), new Embedding(Model, "b", "h2", Unit(0, 1, 0))], CancellationToken.None);
|
||||
await index.RemoveAsync(Model, ["a", "missing"], CancellationToken.None);
|
||||
var state = await index.DescribeAsync(Model, CancellationToken.None);
|
||||
Assert.Equal(["b"], state.ContentHashBySlug.Keys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ModelsAreIsolatedNamespaces()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
await index.UpsertAsync([new Embedding("m1", "a", "h1", Unit(1, 0, 0))], CancellationToken.None);
|
||||
await index.UpsertAsync([new Embedding("m2", "b", "h2", Unit(1, 0, 0))], CancellationToken.None);
|
||||
Assert.Equal(["a"], (await index.QueryAsync("m1", Unit(1, 0, 0), 10, CancellationToken.None)).Select(h => h.Slug));
|
||||
Assert.Equal(["b"], (await index.QueryAsync("m2", Unit(1, 0, 0), 10, CancellationToken.None)).Select(h => h.Slug));
|
||||
Assert.Equal(1, (await index.DescribeAsync("m1", CancellationToken.None)).Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SurvivesFlushAndReopen()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0)), new Embedding(Model, "b", "h2", Unit(0, 1, 0))], CancellationToken.None);
|
||||
await index.FlushAsync(CancellationToken.None);
|
||||
var reopened = await ReopenAsync(index);
|
||||
var state = await reopened.DescribeAsync(Model, CancellationToken.None);
|
||||
Assert.Equal(2, state.Count);
|
||||
var hits = await reopened.QueryAsync(Model, Unit(0, 1, 0), 1, CancellationToken.None);
|
||||
Assert.Equal("b", hits[0].Slug);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthReportsOk()
|
||||
{
|
||||
var index = await CreateIndexAsync();
|
||||
var health = await index.CheckAsync(CancellationToken.None);
|
||||
Assert.True(health.Ok, health.Detail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
|
||||
namespace Wiki.Core.Tests;
|
||||
|
||||
public class EmbeddingTextTests
|
||||
{
|
||||
[Fact]
|
||||
public void ComposesSlugTitleAliasesTagsAndFirstParagraph()
|
||||
{
|
||||
var text = "---\ntitle: UDM intercepts DNS\naliases: [udm dns, split horizon]\ntags: [dns, udm]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers\nevery query.\n\nSecond paragraph.\n";
|
||||
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md", "h"), text);
|
||||
Assert.Equal("udm dns UDM intercepts DNS udm dns split horizon dns udm The UDM answers every query.", EmbeddingText.For(doc));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FirstParagraphNeedsAnH1AndSkipsHeadings()
|
||||
{
|
||||
Assert.Equal(string.Empty, EmbeddingText.FirstParagraph("No heading\n\nText."));
|
||||
Assert.Equal("Text. More.", EmbeddingText.FirstParagraph("# T\n\n## Sub\n\nText.\nMore.\n\nNext."));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CapsLength()
|
||||
{
|
||||
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "x", "wiki/x.md", "h"), "# x\n\n" + new string('a', 5000));
|
||||
Assert.Equal(1500, EmbeddingText.For(doc).Length);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
|
||||
namespace Wiki.Core.Tests;
|
||||
|
||||
public class FrontmatterTests
|
||||
{
|
||||
private const string Note = """
|
||||
---
|
||||
title: UDM intercepts DNS
|
||||
type: wiki
|
||||
tags: [dns, udm, "networking, lan"]
|
||||
aliases:
|
||||
- udm dns
|
||||
- 'split horizon'
|
||||
sources: [raw/_processed/a.md, "live check 2026-09-10: dig @1.1.1.1, dig @10.2.40.99"]
|
||||
related: [ceph-osd, k8s-gateway-in-cluster-dns]
|
||||
updated: 2026-09-16
|
||||
empty:
|
||||
---
|
||||
|
||||
# UDM intercepts DNS
|
||||
|
||||
Body line one.
|
||||
""";
|
||||
|
||||
[Fact]
|
||||
public void ParsesScalarsFlowListsBlockListsAndQuotes()
|
||||
{
|
||||
var (fields, body, start) = Frontmatter.Parse(Note);
|
||||
Assert.Equal("UDM intercepts DNS", fields["title"].Scalar);
|
||||
Assert.Equal(["dns", "udm", "networking, lan"], fields["tags"].List!);
|
||||
Assert.Equal(["udm dns", "split horizon"], fields["aliases"].List!);
|
||||
Assert.Equal(["raw/_processed/a.md", "live check 2026-09-10: dig @1.1.1.1, dig @10.2.40.99"], fields["sources"].List!);
|
||||
Assert.Equal(string.Empty, fields["empty"].Scalar);
|
||||
Assert.Equal(13, start);
|
||||
Assert.StartsWith("\n# UDM intercepts DNS", body);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NoFrontmatterIsWholeBody()
|
||||
{
|
||||
var (fields, body, start) = Frontmatter.Parse("# Just a body\n");
|
||||
Assert.Empty(fields);
|
||||
Assert.Equal("# Just a body\n", body);
|
||||
Assert.Equal(1, start);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnterminatedFrontmatterIsWholeBody()
|
||||
{
|
||||
var (fields, body, _) = Frontmatter.Parse("---\ntitle: x\nno end");
|
||||
Assert.Empty(fields);
|
||||
Assert.Equal("---\ntitle: x\nno end", body);
|
||||
}
|
||||
}
|
||||
|
||||
public class DocumentParserTests
|
||||
{
|
||||
[Fact]
|
||||
public void LiftsVaultFields()
|
||||
{
|
||||
var reference = new DocumentRef(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md", "abc");
|
||||
var doc = DocumentParser.Parse(reference, FrontmatterTests_Note.Text);
|
||||
Assert.Equal("UDM intercepts DNS", doc.Title);
|
||||
Assert.Equal("wiki", doc.Type);
|
||||
Assert.Equal(new DateOnly(2026, 9, 16), doc.Updated);
|
||||
Assert.Equal(["ceph-osd", "k8s-gateway-in-cluster-dns"], doc.Related);
|
||||
Assert.Equal(2, doc.Sources.Count);
|
||||
Assert.Equal(reference, doc.Ref);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TitleFallsBackToSlug()
|
||||
{
|
||||
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Raw, "x", "raw/x.md", "h"), "no frontmatter");
|
||||
Assert.Equal("x", doc.Title);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("wiki/a-b.md", DocumentKind.Wiki, "a-b")]
|
||||
[InlineData("raw/2026-09-23-x.md", DocumentKind.Raw, "2026-09-23-x")]
|
||||
[InlineData("raw/_processed/x.md", DocumentKind.RawProcessed, "x")]
|
||||
[InlineData("output/2026-09-23-y.md", DocumentKind.Output, "2026-09-23-y")]
|
||||
public void ClassifiesVaultPaths(string path, DocumentKind kind, string slug)
|
||||
{
|
||||
Assert.Equal((kind, slug), DocumentParser.Classify(path));
|
||||
Assert.Equal(path, DocumentParser.PathFor(kind, slug));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("wiki/_index.md")]
|
||||
[InlineData("bin/wiki_rank.py")]
|
||||
[InlineData("Untitled.md")]
|
||||
[InlineData("_archive/old.md")]
|
||||
[InlineData("wiki/sub/x.md")]
|
||||
public void IgnoresNonDocuments(string path) => Assert.Null(DocumentParser.Classify(path));
|
||||
}
|
||||
|
||||
public class WikiLinksTests
|
||||
{
|
||||
[Fact]
|
||||
public void ExtractsBodyLinksOutsideCodeAndRelated()
|
||||
{
|
||||
var text = "---\ntitle: T\nrelated: [rel-one, not a slug]\n---\n\nSee [[target-a]] and [[target-b|alias]] and [[target-c#heading]].\n`[[in-code]]`\n```\n[[fenced]]\n```\n[[ -f \"$X\" ]] and [[396,{{value|int}}]]\n";
|
||||
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "src", "wiki/src.md", "h"), text);
|
||||
var links = WikiLinks.Extract(doc);
|
||||
Assert.Equal(
|
||||
[("rel-one", LinkKind.Related, (int?)null), ("target-a", LinkKind.WikiLink, 6), ("target-b", LinkKind.WikiLink, 6), ("target-c", LinkKind.WikiLink, 6)],
|
||||
links.Select(l => (l.ToSlug, l.Kind, l.Line)).ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
public class ParagraphChunkerTests
|
||||
{
|
||||
[Fact]
|
||||
public void SplitsOnBlankLinesHeadingsAndFences()
|
||||
{
|
||||
var text = "---\ntitle: UDM\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n\n## Details\n\nEven dig.\nSecond line.\n\n```\ndig @10.2.40.99\n```\n";
|
||||
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "udm", "wiki/udm.md", "h"), text);
|
||||
var ps = new ParagraphChunker().Split(doc);
|
||||
Assert.Equal(3, ps.Count);
|
||||
Assert.Equal("udm#p1", ps[0].Id);
|
||||
Assert.Equal("UDM intercepts DNS", ps[0].Heading);
|
||||
Assert.Equal("Details", ps[1].Heading);
|
||||
Assert.Contains("Second line", ps[1].Text);
|
||||
Assert.StartsWith("```", ps[2].Text);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class FrontmatterTests_Note
|
||||
{
|
||||
public const string Text = """
|
||||
---
|
||||
title: UDM intercepts DNS
|
||||
type: wiki
|
||||
tags: [dns, udm]
|
||||
sources: [raw/_processed/a.md, "live check: a, b"]
|
||||
related: [ceph-osd, k8s-gateway-in-cluster-dns]
|
||||
updated: 2026-09-16
|
||||
---
|
||||
|
||||
# UDM intercepts DNS
|
||||
""";
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The host with every adapter replaced by a fake: no network, no files. Exercises the wiring,
|
||||
/// the endpoint contracts and the auth handlers.
|
||||
/// </summary>
|
||||
public sealed class WikiHostFactory : WebApplicationFactory<Program>
|
||||
{
|
||||
public const string WebhookSecret = "hook-secret";
|
||||
public const string ApiKey = "machine-key";
|
||||
public FakeStore Store { get; } = new();
|
||||
|
||||
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
||||
{
|
||||
builder.UseEnvironment("Testing");
|
||||
builder.UseSetting("Wiki:Webhook:Secret", WebhookSecret);
|
||||
builder.UseSetting("Auth:ApiKeys:tester", ApiKey);
|
||||
builder.UseSetting("Auth:ReverseProxy:TrustedProxies:0", "127.0.0.1");
|
||||
builder.UseSetting("Wiki:Index:File:Directory", Path.Combine(Path.GetTempPath(), "wiki-host-tests", Guid.NewGuid().ToString("N")));
|
||||
builder.ConfigureServices(services =>
|
||||
{
|
||||
services.RemoveAll<INoteStore>();
|
||||
services.AddSingleton<INoteStore>(Store);
|
||||
services.RemoveAll<IFactSink>();
|
||||
services.AddSingleton<IFactSink>(new FakeFactSink());
|
||||
// The embedding model registered under the active name is replaced by a deterministic fake.
|
||||
services.RemoveAll(typeof(IEmbeddingModel));
|
||||
var fake = new FakeEmbeddings("bge-m3");
|
||||
services.AddKeyedSingleton<IEmbeddingModel>("bge-m3", fake);
|
||||
services.AddSingleton<IEmbeddingModel>(fake);
|
||||
services.AddSingleton<IReadOnlyList<IEmbeddingModel>>([fake]);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public class EndpointTests(WikiHostFactory factory) : IClassFixture<WikiHostFactory>
|
||||
{
|
||||
private HttpClient Client => factory.CreateClient();
|
||||
|
||||
[Fact]
|
||||
public async Task HealthIsAlwaysOk()
|
||||
{
|
||||
var r = await Client.GetAsync("/health");
|
||||
Assert.Equal(HttpStatusCode.OK, r.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReadyIs503UntilSyncedAndOkAfter()
|
||||
{
|
||||
var first = await Client.GetAsync("/ready");
|
||||
// The background sync may or may not have completed; force one and check the steady state.
|
||||
factory.Store.Files["wiki/a.md"] = "---\ntitle: A\ntype: wiki\ntags: [x]\n---\n\n# A\n\nBody.\n";
|
||||
var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true");
|
||||
req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
|
||||
var sync = await Client.SendAsync(req);
|
||||
Assert.Equal(HttpStatusCode.OK, sync.StatusCode);
|
||||
var ready = await Client.GetAsync("/ready");
|
||||
Assert.Equal(HttpStatusCode.OK, ready.StatusCode);
|
||||
Assert.True(first.StatusCode is HttpStatusCode.OK or HttpStatusCode.ServiceUnavailable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SyncRequiresAnApiKey()
|
||||
{
|
||||
var r = await Client.PostAsync("/api/v1/sync", null);
|
||||
Assert.True(r.StatusCode == HttpStatusCode.Unauthorized, $"{(int)r.StatusCode}: {await r.Content.ReadAsStringAsync()}");
|
||||
var bad = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync");
|
||||
bad.Headers.Add("X-Api-Key", "wrong");
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, (await Client.SendAsync(bad)).StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RankRejectsEmptyQueryAndUnknownModel()
|
||||
{
|
||||
var empty = await Client.PostAsJsonAsync("/rank", new { query = " " });
|
||||
Assert.Equal(HttpStatusCode.BadRequest, empty.StatusCode);
|
||||
var unknown = await Client.PostAsJsonAsync("/rank", new { query = "x", model = "nope" });
|
||||
Assert.Equal(HttpStatusCode.BadRequest, unknown.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RankReturnsRowsInThePythonShape()
|
||||
{
|
||||
factory.Store.Files["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns]\n---\n\n# UDM intercepts DNS\n\nThe UDM answers.\n";
|
||||
var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/sync?force=true");
|
||||
req.Headers.Add("X-Api-Key", WikiHostFactory.ApiKey);
|
||||
await Client.SendAsync(req);
|
||||
|
||||
var r = await Client.PostAsJsonAsync("/rank", new { query = "udm dns", k = 3 });
|
||||
Assert.Equal(HttpStatusCode.OK, r.StatusCode);
|
||||
var body = await r.Content.ReadFromJsonAsync<RankResponse>();
|
||||
Assert.NotNull(body);
|
||||
Assert.Contains(body.Rows, row => row.Slug == "udm-dns" && row.Title == "UDM intercepts DNS");
|
||||
Assert.Equal("bge-m3", body.Model);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WebhookVerifiesTheGiteaSignature()
|
||||
{
|
||||
var payload = Encoding.UTF8.GetBytes("{\"ref\":\"refs/heads/main\"}");
|
||||
var good = Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(WikiHostFactory.WebhookSecret), payload));
|
||||
|
||||
var ok = new HttpRequestMessage(HttpMethod.Post, "/webhook") { Content = new ByteArrayContent(payload) };
|
||||
ok.Headers.Add("X-Gitea-Signature", good);
|
||||
Assert.Equal(HttpStatusCode.OK, (await Client.SendAsync(ok)).StatusCode);
|
||||
|
||||
var bad = new HttpRequestMessage(HttpMethod.Post, "/webhook") { Content = new ByteArrayContent(payload) };
|
||||
bad.Headers.Add("X-Gitea-Signature", "deadbeef");
|
||||
Assert.Equal(HttpStatusCode.Forbidden, (await Client.SendAsync(bad)).StatusCode);
|
||||
|
||||
var other = Encoding.UTF8.GetBytes("{\"ref\":\"refs/heads/feature\"}");
|
||||
var otherSig = Convert.ToHexStringLower(HMACSHA256.HashData(Encoding.UTF8.GetBytes(WikiHostFactory.WebhookSecret), other));
|
||||
var skip = new HttpRequestMessage(HttpMethod.Post, "/webhook") { Content = new ByteArrayContent(other) };
|
||||
skip.Headers.Add("X-Gitea-Signature", otherSig);
|
||||
var skipped = await (await Client.SendAsync(skip)).Content.ReadFromJsonAsync<WebhookResponse>();
|
||||
Assert.False(skipped!.Queued);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StatusListsAdaptersAndIndexes()
|
||||
{
|
||||
var status = await Client.GetFromJsonAsync<StatusResponse>("/api/v1/status");
|
||||
Assert.NotNull(status);
|
||||
Assert.Contains(status.Indexes, i => i.ModelId == "bge-m3" && i.Active);
|
||||
Assert.NotEmpty(status.Adapters);
|
||||
}
|
||||
|
||||
private sealed record RankRow(string Slug, string Title, float Score);
|
||||
private sealed record RankResponse(List<RankRow> Rows, string Model);
|
||||
private sealed record WebhookResponse(bool Queued);
|
||||
private sealed record IndexRow(string ModelId, int Notes, bool Active);
|
||||
private sealed record AdapterRow(string Adapter, bool Ok);
|
||||
private sealed record StatusResponse(bool Ready, List<IndexRow> Indexes, List<AdapterRow> Adapters);
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Wiki.Core.Model;
|
||||
using Wiki.Core.Parsing;
|
||||
using Wiki.Core.Ports;
|
||||
|
||||
namespace Wiki.Host.Tests;
|
||||
|
||||
public sealed class FakeStore : INoteStore
|
||||
{
|
||||
public Dictionary<string, string> Files { get; } = new();
|
||||
public string Name => "fake-store";
|
||||
private int _version;
|
||||
|
||||
public Task<StoreHead> GetHeadAsync(CancellationToken ct)
|
||||
{
|
||||
// The head changes whenever the files do, like a repository would.
|
||||
var digest = SHA1.HashData(Encoding.UTF8.GetBytes(string.Join("\n", Files.OrderBy(f => f.Key).Select(f => f.Key + f.Value))));
|
||||
_version++;
|
||||
return Task.FromResult(new StoreHead(Convert.ToHexStringLower(digest), null));
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DocumentRef>> ListAsync(string commit, IReadOnlySet<DocumentKind> kinds, CancellationToken ct)
|
||||
{
|
||||
var refs = new List<DocumentRef>();
|
||||
foreach (var (path, text) in Files)
|
||||
{
|
||||
var c = DocumentParser.Classify(path);
|
||||
if (c is null || !kinds.Contains(c.Value.Kind)) continue;
|
||||
refs.Add(new DocumentRef(c.Value.Kind, c.Value.Slug, path, Convert.ToHexString(SHA1.HashData(Encoding.UTF8.GetBytes(text)))));
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<DocumentRef>>(refs);
|
||||
}
|
||||
|
||||
public Task<string> ReadAsync(DocumentRef reference, string commit, CancellationToken ct) => Task.FromResult(Files[reference.Path]);
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public sealed class FakeFactSink : IFactSink
|
||||
{
|
||||
public string Name => "fake-facts";
|
||||
public Task<FactReceipt> AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct) =>
|
||||
Task.FromResult(new FactReceipt("raw/fake.md", "c", now));
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
|
||||
public sealed class FakeEmbeddings(string modelId) : IEmbeddingModel
|
||||
{
|
||||
public string ModelId { get; } = modelId;
|
||||
public int Dimension => 4;
|
||||
public string Name => "fake-embeddings:" + ModelId;
|
||||
|
||||
public Task<IReadOnlyList<float[]>> EmbedAsync(IReadOnlyList<string> texts, CancellationToken ct)
|
||||
{
|
||||
var vectors = texts.Select(t =>
|
||||
{
|
||||
var h = SHA256.HashData(Encoding.UTF8.GetBytes(t));
|
||||
var v = new float[] { h[0] + 1, h[1] + 1, h[2] + 1, h[3] + 1 };
|
||||
var n = MathF.Sqrt(v.Sum(x => x * x));
|
||||
return v.Select(x => x / n).ToArray();
|
||||
}).ToList();
|
||||
return Task.FromResult<IReadOnlyList<float[]>>(vectors);
|
||||
}
|
||||
|
||||
public Task<AdapterHealth> CheckAsync(CancellationToken ct) => Task.FromResult(new AdapterHealth(Name, true, null, TimeSpan.Zero, DateTimeOffset.UnixEpoch));
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.12" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||
|
||||
Reference in New Issue
Block a user