core: domain model, ports, vault parsing; contract suites for note store, vector index, catalog, embeddings
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,10 @@
|
|||||||
|
<Project>
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||||
|
<LangVersion>latest</LangVersion>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
</PropertyGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
namespace Wiki.Core.Model;
|
||||||
|
|
||||||
|
/// <summary>A paragraph handed to the answer model as evidence, with the judge's probability that it answers.</summary>
|
||||||
|
public sealed record EvidenceParagraph(Paragraph Paragraph, string NoteTitle, double Probability);
|
||||||
|
|
||||||
|
/// <summary>Everything the answer model is allowed to see. Not the whole context: the selected evidence and the judgements.</summary>
|
||||||
|
public sealed record AnswerRequest(
|
||||||
|
string Question,
|
||||||
|
IReadOnlyList<EvidenceParagraph> Evidence,
|
||||||
|
IReadOnlyDictionary<string, Judgement> Judgements,
|
||||||
|
double Covered);
|
||||||
|
|
||||||
|
public abstract record AnswerChunk
|
||||||
|
{
|
||||||
|
/// <summary>A piece of the answer text, in order.</summary>
|
||||||
|
public sealed record Text(string Delta) : AnswerChunk;
|
||||||
|
|
||||||
|
/// <summary>The end of the answer, with what it cost.</summary>
|
||||||
|
public sealed record Done(string ModelId, long InputTokens, long OutputTokens, TimeSpan Elapsed) : AnswerChunk;
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
namespace Wiki.Core.Model;
|
||||||
|
|
||||||
|
/// <summary>Where in the vault a document lives. Kinds are disjoint sets of paths.</summary>
|
||||||
|
public enum DocumentKind
|
||||||
|
{
|
||||||
|
/// <summary>wiki/*.md — the retrieval corpus, embedded.</summary>
|
||||||
|
Wiki,
|
||||||
|
/// <summary>raw/*.md — sources waiting for distillation.</summary>
|
||||||
|
Raw,
|
||||||
|
/// <summary>raw/_processed/*.md — sources distillation has finished with; provenance.</summary>
|
||||||
|
RawProcessed,
|
||||||
|
/// <summary>output/*.md — synthesised answers, audit and consolidation reports.</summary>
|
||||||
|
Output,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A listing entry: enough to know whether a document changed without reading it.</summary>
|
||||||
|
public sealed record DocumentRef(DocumentKind Kind, string Slug, string Path, string ContentHash);
|
||||||
|
|
||||||
|
/// <summary>A parsed vault document. Frontmatter fields the vault uses are lifted into properties; the rest stay in <see cref="Frontmatter"/>.</summary>
|
||||||
|
public sealed record Document(
|
||||||
|
DocumentKind Kind,
|
||||||
|
string Slug,
|
||||||
|
string Path,
|
||||||
|
string ContentHash,
|
||||||
|
string Title,
|
||||||
|
string? Type,
|
||||||
|
IReadOnlyList<string> Tags,
|
||||||
|
IReadOnlyList<string> Aliases,
|
||||||
|
IReadOnlyList<string> Related,
|
||||||
|
IReadOnlyList<string> Sources,
|
||||||
|
DateOnly? Updated,
|
||||||
|
IReadOnlyDictionary<string, FrontmatterValue> Frontmatter,
|
||||||
|
string Body,
|
||||||
|
int BodyStartLine)
|
||||||
|
{
|
||||||
|
public DocumentRef Ref => new(Kind, Slug, Path, ContentHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A frontmatter value: a scalar or a list of scalars. The vault uses nothing deeper.</summary>
|
||||||
|
public sealed record FrontmatterValue(string? Scalar, IReadOnlyList<string>? List)
|
||||||
|
{
|
||||||
|
public static FrontmatterValue Of(string scalar) => new(scalar, null);
|
||||||
|
public static FrontmatterValue Of(IReadOnlyList<string> list) => new(null, list);
|
||||||
|
public bool IsList => List is not null;
|
||||||
|
|
||||||
|
/// <summary>The value as a list: a list as is, a scalar as a one-element list, an empty scalar as empty.</summary>
|
||||||
|
public IReadOnlyList<string> AsList() =>
|
||||||
|
List ?? (string.IsNullOrWhiteSpace(Scalar) ? [] : [Scalar!]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Part of a document body addressable in typed questions: <c>slug#pN</c>.</summary>
|
||||||
|
public sealed record Paragraph(string Id, string Slug, string Heading, string Text);
|
||||||
|
|
||||||
|
/// <summary>An outgoing reference from one document to another, by target slug.</summary>
|
||||||
|
public sealed record Link(string FromSlug, string ToSlug, LinkKind Kind, int? Line);
|
||||||
|
|
||||||
|
public enum LinkKind
|
||||||
|
{
|
||||||
|
/// <summary>[[target]] in the body.</summary>
|
||||||
|
WikiLink,
|
||||||
|
/// <summary>An entry in the <c>related:</c> frontmatter list.</summary>
|
||||||
|
Related,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A caller identity for attribution. Resolved by the host's authentication, carried into use cases.</summary>
|
||||||
|
public sealed record Identity(string Name, IdentityKind Kind)
|
||||||
|
{
|
||||||
|
public static readonly Identity Anonymous = new("anonymous", IdentityKind.Anonymous);
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum IdentityKind
|
||||||
|
{
|
||||||
|
Anonymous,
|
||||||
|
Human,
|
||||||
|
Machine,
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
namespace Wiki.Core.Model;
|
||||||
|
|
||||||
|
/// <summary>Result of an adapter's self check. Reported in status, never used for readiness.</summary>
|
||||||
|
public sealed record AdapterHealth(string Adapter, bool Ok, string? Detail, TimeSpan Latency, DateTimeOffset CheckedAt);
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
namespace Wiki.Core.Model;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A typed question over a state. Three primitives, as TypeSafe's System One defines them;
|
||||||
|
/// any judge adapter has to answer all three.
|
||||||
|
/// </summary>
|
||||||
|
public abstract record TypedQuestion(string Instructions)
|
||||||
|
{
|
||||||
|
/// <summary>A yes/no judgement with a probability. Criteria describe both outcomes.</summary>
|
||||||
|
public sealed record Noul(string Instructions, string TrueCriterion, string FalseCriterion) : TypedQuestion(Instructions);
|
||||||
|
|
||||||
|
/// <summary>One option out of many, with a probability per option. Cap is the judge's concern.</summary>
|
||||||
|
public sealed record Choice(string Instructions, IReadOnlyDictionary<string, string> Options) : TypedQuestion(Instructions);
|
||||||
|
|
||||||
|
/// <summary>A graded judgement in [0, 1]. Criteria describe the ends of the scale.</summary>
|
||||||
|
public sealed record Score(string Instructions, string Criteria) : TypedQuestion(Instructions);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The state a judge sees plus the questions asked about it, one request.</summary>
|
||||||
|
public sealed record JudgeRequest(object State, IReadOnlyDictionary<string, TypedQuestion> Questions);
|
||||||
|
|
||||||
|
public abstract record Judgement
|
||||||
|
{
|
||||||
|
public sealed record Noul(double Probability) : Judgement;
|
||||||
|
public sealed record Choice(string Selected, IReadOnlyDictionary<string, double> Probabilities, double Confidence) : Judgement;
|
||||||
|
public sealed record Score(double Value) : Judgement;
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record JudgeUsage(int Requests, long InputTokens, long OutputTokens, TimeSpan Elapsed)
|
||||||
|
{
|
||||||
|
public static readonly JudgeUsage None = new(0, 0, 0, TimeSpan.Zero);
|
||||||
|
public JudgeUsage Add(JudgeUsage other) =>
|
||||||
|
new(Requests + other.Requests, InputTokens + other.InputTokens, OutputTokens + other.OutputTokens, Elapsed + other.Elapsed);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record JudgeResult(IReadOnlyDictionary<string, Judgement> Answers, JudgeUsage Usage);
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
namespace Wiki.Core.Model;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A fact an agent deposits. Becomes one file in raw/ with provenance frontmatter.
|
||||||
|
/// There is deliberately no field for a hypothesis: the shape enforces the raw rule.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record RawSource(
|
||||||
|
string Title,
|
||||||
|
string Source,
|
||||||
|
string Content,
|
||||||
|
DateTimeOffset ObservedAt,
|
||||||
|
IReadOnlyList<string> Tags)
|
||||||
|
{
|
||||||
|
public const int MinimumContentLength = 40;
|
||||||
|
|
||||||
|
/// <summary>Validation errors, empty when the fact may be stored.</summary>
|
||||||
|
public IReadOnlyList<string> Validate()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
if (string.IsNullOrWhiteSpace(Title)) errors.Add("title is required");
|
||||||
|
if (string.IsNullOrWhiteSpace(Source)) errors.Add("source is required: where the fact was observed or quoted from");
|
||||||
|
if (Content is null || Content.Trim().Length < MinimumContentLength)
|
||||||
|
errors.Add($"content must be at least {MinimumContentLength} characters of observation");
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>What the fact sink returns once the source is stored.</summary>
|
||||||
|
public sealed record FactReceipt(string Path, string Commit, DateTimeOffset StoredAt);
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
namespace Wiki.Core.Model;
|
||||||
|
|
||||||
|
/// <summary>A vector for one document under one embedding model.</summary>
|
||||||
|
public sealed record Embedding(string ModelId, string Slug, string ContentHash, float[] Vector);
|
||||||
|
|
||||||
|
/// <summary>One result of a vector query.</summary>
|
||||||
|
public sealed record RankedHit(string Slug, float Score);
|
||||||
|
|
||||||
|
/// <summary>What an index holds for one model: which documents at which content hash.</summary>
|
||||||
|
public sealed record IndexState(string ModelId, IReadOnlyDictionary<string, string> ContentHashBySlug)
|
||||||
|
{
|
||||||
|
public int Count => ContentHashBySlug.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Head of the note store: the commit the listing is taken from.</summary>
|
||||||
|
public sealed record StoreHead(string Commit, DateTimeOffset? CommittedAt);
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using Wiki.Core.Model;
|
||||||
|
|
||||||
|
namespace Wiki.Core.Parsing;
|
||||||
|
|
||||||
|
/// <summary>Turns a vault file into a <see cref="Document"/>. Kind and slug come from the path.</summary>
|
||||||
|
public static class DocumentParser
|
||||||
|
{
|
||||||
|
public static Document Parse(DocumentRef reference, string text)
|
||||||
|
{
|
||||||
|
var (fields, body, bodyStart) = Frontmatter.Parse(text);
|
||||||
|
var title = Field(fields, "title")?.Scalar?.Trim();
|
||||||
|
if (string.IsNullOrEmpty(title)) title = reference.Slug;
|
||||||
|
return new Document(
|
||||||
|
reference.Kind,
|
||||||
|
reference.Slug,
|
||||||
|
reference.Path,
|
||||||
|
reference.ContentHash,
|
||||||
|
title,
|
||||||
|
Field(fields, "type")?.Scalar?.Trim(),
|
||||||
|
ListField(fields, "tags"),
|
||||||
|
ListField(fields, "aliases"),
|
||||||
|
ListField(fields, "related"),
|
||||||
|
ListField(fields, "sources"),
|
||||||
|
ParseDate(Field(fields, "updated")?.Scalar),
|
||||||
|
fields,
|
||||||
|
body,
|
||||||
|
bodyStart);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Kind and slug for a repository path, or null when the path is not a vault document.</summary>
|
||||||
|
public static (DocumentKind Kind, string Slug)? Classify(string path)
|
||||||
|
{
|
||||||
|
if (!path.EndsWith(".md", StringComparison.Ordinal)) return null;
|
||||||
|
var slash = path.LastIndexOf('/');
|
||||||
|
var dir = slash < 0 ? string.Empty : path[..(slash + 1)];
|
||||||
|
var name = path[(slash + 1)..^3];
|
||||||
|
if (name.Length == 0 || name.StartsWith('_')) return null; // _index.md and friends
|
||||||
|
return dir switch
|
||||||
|
{
|
||||||
|
"wiki/" => (DocumentKind.Wiki, name),
|
||||||
|
"raw/" => (DocumentKind.Raw, name),
|
||||||
|
"raw/_processed/" => (DocumentKind.RawProcessed, name),
|
||||||
|
"output/" => (DocumentKind.Output, name),
|
||||||
|
_ => null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string PathFor(DocumentKind kind, string slug) => kind switch
|
||||||
|
{
|
||||||
|
DocumentKind.Wiki => $"wiki/{slug}.md",
|
||||||
|
DocumentKind.Raw => $"raw/{slug}.md",
|
||||||
|
DocumentKind.RawProcessed => $"raw/_processed/{slug}.md",
|
||||||
|
DocumentKind.Output => $"output/{slug}.md",
|
||||||
|
_ => throw new ArgumentOutOfRangeException(nameof(kind)),
|
||||||
|
};
|
||||||
|
|
||||||
|
private static FrontmatterValue? Field(IReadOnlyDictionary<string, FrontmatterValue> fields, string key) =>
|
||||||
|
fields.TryGetValue(key, out var v) ? v : null;
|
||||||
|
|
||||||
|
private static IReadOnlyList<string> ListField(IReadOnlyDictionary<string, FrontmatterValue> fields, string key) =>
|
||||||
|
Field(fields, key)?.AsList() ?? [];
|
||||||
|
|
||||||
|
public static DateOnly? ParseDate(string? value)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(value)) return null;
|
||||||
|
var s = value.Trim();
|
||||||
|
if (s.Length >= 10 && DateOnly.TryParseExact(s[..10], "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var d))
|
||||||
|
return d;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
using System.Text;
|
||||||
|
using Wiki.Core.Model;
|
||||||
|
|
||||||
|
namespace Wiki.Core.Parsing;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads the YAML subset the vault uses: <c>key: scalar</c>, <c>key: [a, "b, c", 'd']</c>,
|
||||||
|
/// block lists of <c>- item</c>, quoted scalars, and nothing nested deeper. Tolerant: a line
|
||||||
|
/// it cannot read is skipped, never fatal, because a note with odd frontmatter is still a note.
|
||||||
|
/// </summary>
|
||||||
|
public static class Frontmatter
|
||||||
|
{
|
||||||
|
public static (IReadOnlyDictionary<string, FrontmatterValue> Fields, string Body, int BodyStartLine) Parse(string text)
|
||||||
|
{
|
||||||
|
var fields = new Dictionary<string, FrontmatterValue>(StringComparer.Ordinal);
|
||||||
|
if (!text.StartsWith("---", StringComparison.Ordinal))
|
||||||
|
return (fields, text, 1);
|
||||||
|
|
||||||
|
var lines = text.Split('\n');
|
||||||
|
if (lines[0].TrimEnd('\r').Trim() != "---")
|
||||||
|
return (fields, text, 1);
|
||||||
|
|
||||||
|
var end = -1;
|
||||||
|
for (var i = 1; i < lines.Length; i++)
|
||||||
|
{
|
||||||
|
if (lines[i].TrimEnd('\r').Trim() == "---") { end = i; break; }
|
||||||
|
}
|
||||||
|
if (end < 0)
|
||||||
|
return (fields, text, 1);
|
||||||
|
|
||||||
|
string? pendingKey = null;
|
||||||
|
List<string>? pendingList = null;
|
||||||
|
for (var i = 1; i < end; i++)
|
||||||
|
{
|
||||||
|
var line = lines[i].TrimEnd('\r');
|
||||||
|
if (string.IsNullOrWhiteSpace(line) || line.TrimStart().StartsWith('#'))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (pendingKey is not null && line.TrimStart().StartsWith("- ", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
pendingList ??= [];
|
||||||
|
pendingList.Add(Unquote(line.TrimStart()[2..].Trim()));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (pendingKey is not null)
|
||||||
|
{
|
||||||
|
fields[pendingKey] = pendingList is null ? FrontmatterValue.Of(string.Empty) : FrontmatterValue.Of(pendingList);
|
||||||
|
pendingKey = null;
|
||||||
|
pendingList = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var colon = IndexOfKeyColon(line);
|
||||||
|
if (colon < 0)
|
||||||
|
continue;
|
||||||
|
var key = line[..colon].Trim();
|
||||||
|
var value = line[(colon + 1)..].Trim();
|
||||||
|
if (key.Length == 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (value.Length == 0)
|
||||||
|
{
|
||||||
|
pendingKey = key; // a block list may follow; otherwise an empty scalar
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (value.StartsWith('[') && value.EndsWith(']'))
|
||||||
|
{
|
||||||
|
fields[key] = FrontmatterValue.Of(SplitFlowList(value[1..^1]));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
fields[key] = FrontmatterValue.Of(Unquote(value));
|
||||||
|
}
|
||||||
|
if (pendingKey is not null)
|
||||||
|
fields[pendingKey] = pendingList is null ? FrontmatterValue.Of(string.Empty) : FrontmatterValue.Of(pendingList);
|
||||||
|
|
||||||
|
var body = string.Join('\n', lines.Skip(end + 1));
|
||||||
|
return (fields, body, end + 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int IndexOfKeyColon(string line)
|
||||||
|
{
|
||||||
|
// A key is [A-Za-z_][\w-]* at column 0 followed by ':' and a space or end of line.
|
||||||
|
var i = 0;
|
||||||
|
if (i >= line.Length || !(char.IsLetter(line[i]) || line[i] == '_')) return -1;
|
||||||
|
while (i < line.Length && (char.IsLetterOrDigit(line[i]) || line[i] == '_' || line[i] == '-')) i++;
|
||||||
|
if (i >= line.Length || line[i] != ':') return -1;
|
||||||
|
if (i + 1 < line.Length && line[i + 1] != ' ' && line[i + 1] != '\t') return -1;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Splits <c>a, "b, c", 'd'</c> respecting quotes; sources lists carry commas inside quotes.</summary>
|
||||||
|
public static IReadOnlyList<string> SplitFlowList(string inner)
|
||||||
|
{
|
||||||
|
var items = new List<string>();
|
||||||
|
var current = new StringBuilder();
|
||||||
|
char? quote = null;
|
||||||
|
foreach (var ch in inner)
|
||||||
|
{
|
||||||
|
if (quote is null && (ch == '"' || ch == '\''))
|
||||||
|
{
|
||||||
|
quote = ch;
|
||||||
|
current.Append(ch);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (quote is not null && ch == quote)
|
||||||
|
{
|
||||||
|
quote = null;
|
||||||
|
current.Append(ch);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (quote is null && ch == ',')
|
||||||
|
{
|
||||||
|
Push(items, current);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
current.Append(ch);
|
||||||
|
}
|
||||||
|
Push(items, current);
|
||||||
|
return items;
|
||||||
|
|
||||||
|
static void Push(List<string> items, StringBuilder sb)
|
||||||
|
{
|
||||||
|
var s = Unquote(sb.ToString().Trim());
|
||||||
|
sb.Clear();
|
||||||
|
if (s.Length > 0) items.Add(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string Unquote(string value)
|
||||||
|
{
|
||||||
|
if (value.Length >= 2 && ((value[0] == '"' && value[^1] == '"') || (value[0] == '\'' && value[^1] == '\'')))
|
||||||
|
return value[1..^1];
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
using System.Text;
|
||||||
|
using Wiki.Core.Model;
|
||||||
|
using Wiki.Core.Ports;
|
||||||
|
|
||||||
|
namespace Wiki.Core.Parsing;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The vault's paragraph splitter, as wiki_ask.py does it: blank-line separated blocks,
|
||||||
|
/// headings set the block's heading and are not blocks themselves, fenced code is one block.
|
||||||
|
/// Ids are <c>slug#pN</c>, counted from 1 in document order.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class ParagraphChunker : IChunker
|
||||||
|
{
|
||||||
|
public IReadOnlyList<Paragraph> Split(Document document)
|
||||||
|
{
|
||||||
|
var paragraphs = new List<Paragraph>();
|
||||||
|
var heading = document.Title;
|
||||||
|
var block = new StringBuilder();
|
||||||
|
var fenced = false;
|
||||||
|
|
||||||
|
void Flush()
|
||||||
|
{
|
||||||
|
var text = block.ToString().Trim();
|
||||||
|
block.Clear();
|
||||||
|
if (text.Length == 0) return;
|
||||||
|
var n = paragraphs.Count + 1;
|
||||||
|
paragraphs.Add(new Paragraph($"{document.Slug}#p{n}", document.Slug, heading, text));
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var raw in document.Body.Split('\n'))
|
||||||
|
{
|
||||||
|
var line = raw.TrimEnd('\r');
|
||||||
|
var trimmed = line.TrimStart();
|
||||||
|
if (trimmed.StartsWith("```", StringComparison.Ordinal) || trimmed.StartsWith("~~~", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
if (!fenced) Flush();
|
||||||
|
fenced = !fenced;
|
||||||
|
block.Append(line).Append('\n');
|
||||||
|
if (!fenced) Flush();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (fenced)
|
||||||
|
{
|
||||||
|
block.Append(line).Append('\n');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (trimmed.StartsWith('#'))
|
||||||
|
{
|
||||||
|
Flush();
|
||||||
|
heading = trimmed.TrimStart('#').Trim();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (string.IsNullOrWhiteSpace(line))
|
||||||
|
{
|
||||||
|
Flush();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
block.Append(line).Append('\n');
|
||||||
|
}
|
||||||
|
Flush();
|
||||||
|
return paragraphs;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using Wiki.Core.Model;
|
||||||
|
|
||||||
|
namespace Wiki.Core.Parsing;
|
||||||
|
|
||||||
|
/// <summary>Outgoing references of a document: [[links]] in the body outside code, plus the related list.</summary>
|
||||||
|
public static partial class WikiLinks
|
||||||
|
{
|
||||||
|
[GeneratedRegex(@"\[\[([^\]\[|#]+)(?:#[^\]\[|]*)?(?:\|[^\]\[]*)?\]\]")]
|
||||||
|
private static partial Regex LinkPattern();
|
||||||
|
|
||||||
|
[GeneratedRegex(@"^[A-Za-z0-9][A-Za-z0-9._-]*$")]
|
||||||
|
private static partial Regex SlugPattern();
|
||||||
|
|
||||||
|
[GeneratedRegex("`[^`]*`")]
|
||||||
|
private static partial Regex InlineCode();
|
||||||
|
|
||||||
|
public static bool IsSlug(string value) => SlugPattern().IsMatch(value);
|
||||||
|
|
||||||
|
public static IReadOnlyList<Link> Extract(Document document)
|
||||||
|
{
|
||||||
|
var links = new List<Link>();
|
||||||
|
foreach (var target in document.Related)
|
||||||
|
{
|
||||||
|
if (SlugPattern().IsMatch(target))
|
||||||
|
links.Add(new Link(document.Slug, target, LinkKind.Related, null));
|
||||||
|
}
|
||||||
|
|
||||||
|
var fenced = false;
|
||||||
|
var lineNo = document.BodyStartLine - 1;
|
||||||
|
foreach (var raw in document.Body.Split('\n'))
|
||||||
|
{
|
||||||
|
lineNo++;
|
||||||
|
var line = raw.TrimEnd('\r');
|
||||||
|
var trimmed = line.TrimStart();
|
||||||
|
if (trimmed.StartsWith("```", StringComparison.Ordinal) || trimmed.StartsWith("~~~", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
fenced = !fenced;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (fenced) continue;
|
||||||
|
var stripped = InlineCode().Replace(line, string.Empty);
|
||||||
|
foreach (Match m in LinkPattern().Matches(stripped))
|
||||||
|
{
|
||||||
|
var target = m.Groups[1].Value.Trim();
|
||||||
|
if (SlugPattern().IsMatch(target))
|
||||||
|
links.Add(new Link(document.Slug, target, LinkKind.WikiLink, lineNo));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return links;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
using Wiki.Core.Model;
|
||||||
|
|
||||||
|
namespace Wiki.Core.Ports;
|
||||||
|
|
||||||
|
/// <summary>Anything that can say whether it is reachable and working. Every adapter implements it.</summary>
|
||||||
|
public interface IAdapterHealth
|
||||||
|
{
|
||||||
|
string Name { get; }
|
||||||
|
Task<AdapterHealth> CheckAsync(CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Where the vault's documents are read from.</summary>
|
||||||
|
public interface INoteStore : IAdapterHealth
|
||||||
|
{
|
||||||
|
/// <summary>The current head; a listing is always taken at a head.</summary>
|
||||||
|
Task<StoreHead> GetHeadAsync(CancellationToken ct);
|
||||||
|
|
||||||
|
/// <summary>Every document of the given kinds at that head, with a content hash that changes iff the content changes.</summary>
|
||||||
|
Task<IReadOnlyList<DocumentRef>> ListAsync(string commit, IReadOnlySet<DocumentKind> kinds, CancellationToken ct);
|
||||||
|
|
||||||
|
/// <summary>The document's text at that head.</summary>
|
||||||
|
Task<string> ReadAsync(DocumentRef reference, string commit, CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Where a deposited fact goes: one new raw source, never an edit of an existing file.</summary>
|
||||||
|
public interface IFactSink : IAdapterHealth
|
||||||
|
{
|
||||||
|
Task<FactReceipt> AppendAsync(RawSource fact, Identity by, DateTimeOffset now, CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Text to vector. One instance per configured model.</summary>
|
||||||
|
public interface IEmbeddingModel : IAdapterHealth
|
||||||
|
{
|
||||||
|
string ModelId { get; }
|
||||||
|
int Dimension { get; }
|
||||||
|
Task<IReadOnlyList<float[]>> EmbedAsync(IReadOnlyList<string> texts, CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Vectors per (model, document) with top-k query. Models are namespaces; nothing is shared across them.</summary>
|
||||||
|
public interface IVectorIndex : IAdapterHealth
|
||||||
|
{
|
||||||
|
Task<IndexState> DescribeAsync(string modelId, CancellationToken ct);
|
||||||
|
Task UpsertAsync(IReadOnlyList<Embedding> embeddings, CancellationToken ct);
|
||||||
|
Task RemoveAsync(string modelId, IReadOnlyList<string> slugs, CancellationToken ct);
|
||||||
|
Task<IReadOnlyList<RankedHit>> QueryAsync(string modelId, float[] query, int k, CancellationToken ct);
|
||||||
|
/// <summary>Persist whatever the adapter keeps in memory. A no-op for adapters that write through.</summary>
|
||||||
|
Task FlushAsync(CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The synced documents, queryable: bodies, frontmatter and the link graph. Filled by the sync, read by everything else.</summary>
|
||||||
|
public interface IDocumentCatalog : IAdapterHealth
|
||||||
|
{
|
||||||
|
Task<CatalogState> GetStateAsync(CancellationToken ct);
|
||||||
|
Task SetStateAsync(CatalogState state, CancellationToken ct);
|
||||||
|
Task UpsertAsync(IReadOnlyList<Document> documents, CancellationToken ct);
|
||||||
|
Task RemoveAsync(IReadOnlyList<DocumentRef> references, CancellationToken ct);
|
||||||
|
Task<Document?> GetAsync(DocumentKind kind, string slug, CancellationToken ct);
|
||||||
|
Task<IReadOnlyList<DocumentRef>> ListAsync(DocumentKind kind, CancellationToken ct);
|
||||||
|
Task<IReadOnlyList<Document>> ListDocumentsAsync(DocumentKind kind, CancellationToken ct);
|
||||||
|
/// <summary>Wiki documents that reference the slug, by wikilink or related entry.</summary>
|
||||||
|
Task<IReadOnlyList<Link>> BacklinksAsync(string slug, CancellationToken ct);
|
||||||
|
Task<IReadOnlyList<Link>> OutgoingLinksAsync(string slug, CancellationToken ct);
|
||||||
|
Task<IReadOnlyDictionary<string, int>> TagCountsAsync(CancellationToken ct);
|
||||||
|
Task FlushAsync(CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Which head the catalog reflects and when it got there.</summary>
|
||||||
|
public sealed record CatalogState(string? Commit, DateTimeOffset? SyncedAt);
|
||||||
|
|
||||||
|
/// <summary>Splits a document body into addressable paragraphs for typed questions.</summary>
|
||||||
|
public interface IChunker
|
||||||
|
{
|
||||||
|
IReadOnlyList<Paragraph> Split(Document document);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Typed questions over a state. Returns probabilities, never prose.</summary>
|
||||||
|
public interface IJudge : IAdapterHealth
|
||||||
|
{
|
||||||
|
string ModelId { get; }
|
||||||
|
/// <summary>The largest number of options one Choice may carry, or null for no cap.</summary>
|
||||||
|
int? MaxChoiceOptions { get; }
|
||||||
|
Task<JudgeResult> JudgeAsync(JudgeRequest request, CancellationToken ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Prose from a question and its evidence. Sees only what the request carries.</summary>
|
||||||
|
public interface IAnswerModel : IAdapterHealth
|
||||||
|
{
|
||||||
|
string ModelId { get; }
|
||||||
|
IAsyncEnumerable<AnswerChunk> AnswerAsync(AnswerRequest request, CancellationToken ct);
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
using Wiki.Core.Model;
|
||||||
|
using Wiki.Core.Parsing;
|
||||||
|
using Wiki.Core.Ports;
|
||||||
|
|
||||||
|
namespace Wiki.Contracts;
|
||||||
|
|
||||||
|
/// <summary>Every <see cref="IDocumentCatalog"/> adapter derives from this.</summary>
|
||||||
|
public abstract class DocumentCatalogContract
|
||||||
|
{
|
||||||
|
protected abstract Task<IDocumentCatalog> CreateCatalogAsync();
|
||||||
|
protected abstract Task<IDocumentCatalog> ReopenAsync(IDocumentCatalog catalog);
|
||||||
|
|
||||||
|
private static Document Doc(string path, string hash = "h") =>
|
||||||
|
DocumentParser.Parse(Ref(path, hash), VaultFixture.Files[path]);
|
||||||
|
|
||||||
|
private static DocumentRef Ref(string path, string hash = "h")
|
||||||
|
{
|
||||||
|
var (kind, slug) = DocumentParser.Classify(path)!.Value;
|
||||||
|
return new DocumentRef(kind, slug, path, hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task StartsEmptyWithNoState()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
var state = await catalog.GetStateAsync(CancellationToken.None);
|
||||||
|
Assert.Null(state.Commit);
|
||||||
|
Assert.Empty(await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task UpsertGetListByKind()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md"), Doc("raw/2026-09-23-pending.md"), Doc("output/2026-09-20-answer.md")], CancellationToken.None);
|
||||||
|
|
||||||
|
var got = await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None);
|
||||||
|
Assert.NotNull(got);
|
||||||
|
Assert.Equal("UDM intercepts DNS", got.Title);
|
||||||
|
Assert.Contains("answers every UDP/53", got.Body);
|
||||||
|
Assert.Null(await catalog.GetAsync(DocumentKind.Raw, "udm-dns", CancellationToken.None));
|
||||||
|
|
||||||
|
Assert.Equal(2, (await catalog.ListAsync(DocumentKind.Wiki, CancellationToken.None)).Count);
|
||||||
|
Assert.Single(await catalog.ListAsync(DocumentKind.Raw, CancellationToken.None));
|
||||||
|
Assert.Single(await catalog.ListDocumentsAsync(DocumentKind.Output, CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task LinksAndBacklinksFollowWikilinksAndRelated()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None);
|
||||||
|
var outgoing = await catalog.OutgoingLinksAsync("udm-dns", CancellationToken.None);
|
||||||
|
Assert.Equal([LinkKind.Related, LinkKind.WikiLink], outgoing.Select(l => l.Kind));
|
||||||
|
Assert.All(outgoing, l => Assert.Equal("ceph-osd", l.ToSlug));
|
||||||
|
var back = await catalog.BacklinksAsync("ceph-osd", CancellationToken.None);
|
||||||
|
Assert.Equal(2, back.Count);
|
||||||
|
Assert.All(back, l => Assert.Equal("udm-dns", l.FromSlug));
|
||||||
|
Assert.Empty(await catalog.BacklinksAsync("udm-dns", CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task RemoveDropsDocumentAndItsLinks()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None);
|
||||||
|
await catalog.RemoveAsync([Ref("wiki/udm-dns.md")], CancellationToken.None);
|
||||||
|
Assert.Null(await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None));
|
||||||
|
Assert.Empty(await catalog.BacklinksAsync("ceph-osd", CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task UpsertReplacesLinks()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
await catalog.UpsertAsync([Doc("wiki/udm-dns.md")], CancellationToken.None);
|
||||||
|
var changed = Doc("wiki/udm-dns.md", "h2") with { Body = "no links any more", Related = [] };
|
||||||
|
await catalog.UpsertAsync([changed], CancellationToken.None);
|
||||||
|
Assert.Empty(await catalog.OutgoingLinksAsync("udm-dns", CancellationToken.None));
|
||||||
|
Assert.Equal("h2", (await catalog.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!.ContentHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task TagCountsCoverWikiOnly()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md"), Doc("output/2026-09-20-answer.md")], CancellationToken.None);
|
||||||
|
var tags = await catalog.TagCountsAsync(CancellationToken.None);
|
||||||
|
Assert.Equal(1, tags["dns"]);
|
||||||
|
Assert.Equal(1, tags["udm"]);
|
||||||
|
Assert.Equal(1, tags["ceph"]);
|
||||||
|
Assert.Equal(3, tags.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task StateAndDocumentsSurviveFlushAndReopen()
|
||||||
|
{
|
||||||
|
var catalog = await CreateCatalogAsync();
|
||||||
|
await catalog.UpsertAsync([Doc("wiki/udm-dns.md"), Doc("wiki/ceph-osd.md")], CancellationToken.None);
|
||||||
|
var at = new DateTimeOffset(2026, 9, 24, 10, 0, 0, TimeSpan.Zero);
|
||||||
|
await catalog.SetStateAsync(new CatalogState(VaultFixture.Commit, at), CancellationToken.None);
|
||||||
|
await catalog.FlushAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
var reopened = await ReopenAsync(catalog);
|
||||||
|
var state = await reopened.GetStateAsync(CancellationToken.None);
|
||||||
|
Assert.Equal(VaultFixture.Commit, state.Commit);
|
||||||
|
Assert.Equal(at, state.SyncedAt);
|
||||||
|
Assert.Equal(2, (await reopened.ListAsync(DocumentKind.Wiki, CancellationToken.None)).Count);
|
||||||
|
Assert.Equal(2, (await reopened.BacklinksAsync("ceph-osd", CancellationToken.None)).Count);
|
||||||
|
Assert.Equal("UDM intercepts DNS", (await reopened.GetAsync(DocumentKind.Wiki, "udm-dns", CancellationToken.None))!.Title);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
using Wiki.Core.Ports;
|
||||||
|
|
||||||
|
namespace Wiki.Contracts;
|
||||||
|
|
||||||
|
/// <summary>Every <see cref="IEmbeddingModel"/> adapter derives from this. The adapter under test may talk to a fake server.</summary>
|
||||||
|
public abstract class EmbeddingModelContract
|
||||||
|
{
|
||||||
|
protected abstract Task<IEmbeddingModel> CreateModelAsync();
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task VectorsHaveTheDeclaredDimensionAndUnitLength()
|
||||||
|
{
|
||||||
|
var model = await CreateModelAsync();
|
||||||
|
var vectors = await model.EmbedAsync(["alpha", "beta"], CancellationToken.None);
|
||||||
|
Assert.Equal(2, vectors.Count);
|
||||||
|
foreach (var v in vectors)
|
||||||
|
{
|
||||||
|
Assert.Equal(model.Dimension, v.Length);
|
||||||
|
var norm = MathF.Sqrt(v.Sum(x => x * x));
|
||||||
|
Assert.InRange(norm, 0.99f, 1.01f);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SameTextSameVectorAndOrderPreserved()
|
||||||
|
{
|
||||||
|
var model = await CreateModelAsync();
|
||||||
|
var once = await model.EmbedAsync(["one", "two", "three"], CancellationToken.None);
|
||||||
|
var again = await model.EmbedAsync(["three", "one"], CancellationToken.None);
|
||||||
|
Assert.Equal(once[2], again[0]);
|
||||||
|
Assert.Equal(once[0], again[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EmptyBatchIsEmpty()
|
||||||
|
{
|
||||||
|
var model = await CreateModelAsync();
|
||||||
|
Assert.Empty(await model.EmbedAsync([], CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ModelIdIsSet()
|
||||||
|
{
|
||||||
|
var model = await CreateModelAsync();
|
||||||
|
Assert.False(string.IsNullOrWhiteSpace(model.ModelId));
|
||||||
|
var health = await model.CheckAsync(CancellationToken.None);
|
||||||
|
Assert.True(health.Ok, health.Detail);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
using Wiki.Core.Model;
|
||||||
|
using Wiki.Core.Ports;
|
||||||
|
|
||||||
|
namespace Wiki.Contracts;
|
||||||
|
|
||||||
|
/// <summary>Every <see cref="INoteStore"/> adapter derives a test class from this and supplies a store over <see cref="VaultFixture"/>.</summary>
|
||||||
|
public abstract class NoteStoreContract
|
||||||
|
{
|
||||||
|
/// <summary>A store whose current head exposes exactly <see cref="VaultFixture.Files"/>.</summary>
|
||||||
|
protected abstract Task<INoteStore> CreateStoreAsync();
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HeadHasACommit()
|
||||||
|
{
|
||||||
|
var store = await CreateStoreAsync();
|
||||||
|
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||||
|
Assert.False(string.IsNullOrWhiteSpace(head.Commit));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ListsOnlyVaultDocumentsOfRequestedKinds()
|
||||||
|
{
|
||||||
|
var store = await CreateStoreAsync();
|
||||||
|
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||||
|
var all = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||||
|
Assert.Equal(
|
||||||
|
VaultFixture.Expected.OrderBy(e => e.Path),
|
||||||
|
all.Select(r => (r.Kind, r.Slug, r.Path)).OrderBy(e => e.Path));
|
||||||
|
|
||||||
|
var wiki = await store.ListAsync(head.Commit, new HashSet<DocumentKind> { DocumentKind.Wiki }, CancellationToken.None);
|
||||||
|
Assert.All(wiki, r => Assert.Equal(DocumentKind.Wiki, r.Kind));
|
||||||
|
Assert.Equal(2, wiki.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ContentHashIsStableAndDistinct()
|
||||||
|
{
|
||||||
|
var store = await CreateStoreAsync();
|
||||||
|
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||||
|
var a = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||||
|
var b = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||||
|
Assert.Equal(a.OrderBy(r => r.Path).Select(r => r.ContentHash), b.OrderBy(r => r.Path).Select(r => r.ContentHash));
|
||||||
|
Assert.All(a, r => Assert.False(string.IsNullOrWhiteSpace(r.ContentHash)));
|
||||||
|
Assert.Equal(a.Count, a.Select(r => r.ContentHash).Distinct().Count());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ReadsExactText()
|
||||||
|
{
|
||||||
|
var store = await CreateStoreAsync();
|
||||||
|
var head = await store.GetHeadAsync(CancellationToken.None);
|
||||||
|
var refs = await store.ListAsync(head.Commit, VaultFixture.AllKinds, CancellationToken.None);
|
||||||
|
foreach (var r in refs)
|
||||||
|
{
|
||||||
|
var text = await store.ReadAsync(r, head.Commit, CancellationToken.None);
|
||||||
|
Assert.Equal(VaultFixture.Files[r.Path], text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HealthReportsOk()
|
||||||
|
{
|
||||||
|
var store = await CreateStoreAsync();
|
||||||
|
var health = await store.CheckAsync(CancellationToken.None);
|
||||||
|
Assert.True(health.Ok, health.Detail);
|
||||||
|
Assert.Equal(store.Name, health.Adapter);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
using Wiki.Core.Model;
|
||||||
|
|
||||||
|
namespace Wiki.Contracts;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A tiny vault every note-store adapter has to expose identically. Adapters under test are
|
||||||
|
/// pointed at a fake or a temp copy holding exactly these files; the contract checks that the
|
||||||
|
/// listing and reads come back as written.
|
||||||
|
/// </summary>
|
||||||
|
public static class VaultFixture
|
||||||
|
{
|
||||||
|
public const string Commit = "0123456789abcdef0123456789abcdef01234567";
|
||||||
|
|
||||||
|
public static readonly IReadOnlyDictionary<string, string> Files = new Dictionary<string, string>
|
||||||
|
{
|
||||||
|
["wiki/_index.md"] = "---\ntitle: index\ntype: index\n---\n",
|
||||||
|
["wiki/udm-dns.md"] = "---\ntitle: UDM intercepts DNS\ntype: wiki\ntags: [dns, udm]\nsources: [raw/_processed/2026-09-01-dns.md]\nrelated: [ceph-osd]\nupdated: 2026-09-16\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every UDP/53 query itself. See [[ceph-osd]].\n",
|
||||||
|
["wiki/ceph-osd.md"] = "---\ntitle: Ceph OSD\ntype: wiki\ntags: [ceph]\nsources: [raw/_processed/2026-09-01-dns.md]\nupdated: 2026-09-10\n---\n\n# Ceph OSD\n\nOSD fsid mismatch after rebuild.\n",
|
||||||
|
["raw/2026-09-23-pending.md"] = "---\ntitle: Pending source\ntype: raw\nsource: test\ncaptured: 2026-09-23\n---\n\nVerbatim capture.\n",
|
||||||
|
["raw/_processed/2026-09-01-dns.md"] = "---\ntitle: Processed source\ntype: raw\nsource: test\ncaptured: 2026-09-01\n---\n\nVerbatim capture, processed.\n",
|
||||||
|
["output/2026-09-20-answer.md"] = "---\ntitle: An answer\ntype: output\nquestion: Does the UDM intercept DNS?\nsources: [udm-dns]\n---\n\nYes.\n",
|
||||||
|
["bin/wiki_rank.py"] = "#!/usr/bin/env python3\n",
|
||||||
|
["Untitled.md"] = "not a vault document\n",
|
||||||
|
};
|
||||||
|
|
||||||
|
/// <summary>The refs a listing over all kinds must return (order not significant).</summary>
|
||||||
|
public static readonly IReadOnlyList<(DocumentKind Kind, string Slug, string Path)> Expected =
|
||||||
|
[
|
||||||
|
(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md"),
|
||||||
|
(DocumentKind.Wiki, "ceph-osd", "wiki/ceph-osd.md"),
|
||||||
|
(DocumentKind.Raw, "2026-09-23-pending", "raw/2026-09-23-pending.md"),
|
||||||
|
(DocumentKind.RawProcessed, "2026-09-01-dns", "raw/_processed/2026-09-01-dns.md"),
|
||||||
|
(DocumentKind.Output, "2026-09-20-answer", "output/2026-09-20-answer.md"),
|
||||||
|
];
|
||||||
|
|
||||||
|
public static readonly IReadOnlySet<DocumentKind> AllKinds =
|
||||||
|
new HashSet<DocumentKind> { DocumentKind.Wiki, DocumentKind.Raw, DocumentKind.RawProcessed, DocumentKind.Output };
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
using Wiki.Core.Model;
|
||||||
|
using Wiki.Core.Ports;
|
||||||
|
|
||||||
|
namespace Wiki.Contracts;
|
||||||
|
|
||||||
|
/// <summary>Every <see cref="IVectorIndex"/> adapter derives from this. Vectors are unit length; scores are cosine.</summary>
|
||||||
|
public abstract class VectorIndexContract
|
||||||
|
{
|
||||||
|
/// <summary>A fresh, empty index. Called again after <see cref="ReopenAsync"/> semantics are needed.</summary>
|
||||||
|
protected abstract Task<IVectorIndex> CreateIndexAsync();
|
||||||
|
|
||||||
|
/// <summary>The same index reopened from persisted state, or the same instance for adapters that keep no files.</summary>
|
||||||
|
protected abstract Task<IVectorIndex> ReopenAsync(IVectorIndex index);
|
||||||
|
|
||||||
|
private const string Model = "test-model";
|
||||||
|
|
||||||
|
private static float[] Unit(params float[] v)
|
||||||
|
{
|
||||||
|
var n = MathF.Sqrt(v.Sum(x => x * x));
|
||||||
|
return v.Select(x => x / n).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EmptyIndexDescribesEmptyAndQueriesNothing()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
var state = await index.DescribeAsync(Model, CancellationToken.None);
|
||||||
|
Assert.Equal(0, state.Count);
|
||||||
|
Assert.Empty(await index.QueryAsync(Model, Unit(1, 0, 0), 5, CancellationToken.None));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task QueryRanksByCosineAndHonoursK()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
await index.UpsertAsync(
|
||||||
|
[
|
||||||
|
new Embedding(Model, "a", "h1", Unit(1, 0, 0)),
|
||||||
|
new Embedding(Model, "b", "h2", Unit(0.9f, 0.1f, 0)),
|
||||||
|
new Embedding(Model, "c", "h3", Unit(0, 1, 0)),
|
||||||
|
new Embedding(Model, "d", "h4", Unit(0, 0, 1)),
|
||||||
|
], CancellationToken.None);
|
||||||
|
|
||||||
|
var hits = await index.QueryAsync(Model, Unit(1, 0, 0), 2, CancellationToken.None);
|
||||||
|
Assert.Equal(["a", "b"], hits.Select(h => h.Slug));
|
||||||
|
Assert.True(hits[0].Score > hits[1].Score);
|
||||||
|
Assert.InRange(hits[0].Score, 0.999f, 1.001f);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task UpsertReplacesAndDescribeTracksHashes()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0))], CancellationToken.None);
|
||||||
|
await index.UpsertAsync([new Embedding(Model, "a", "h2", Unit(0, 1, 0))], CancellationToken.None);
|
||||||
|
var state = await index.DescribeAsync(Model, CancellationToken.None);
|
||||||
|
Assert.Equal(1, state.Count);
|
||||||
|
Assert.Equal("h2", state.ContentHashBySlug["a"]);
|
||||||
|
var hits = await index.QueryAsync(Model, Unit(0, 1, 0), 1, CancellationToken.None);
|
||||||
|
Assert.Equal("a", hits[0].Slug);
|
||||||
|
Assert.InRange(hits[0].Score, 0.999f, 1.001f);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task RemoveDropsSlugs()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0)), new Embedding(Model, "b", "h2", Unit(0, 1, 0))], CancellationToken.None);
|
||||||
|
await index.RemoveAsync(Model, ["a", "missing"], CancellationToken.None);
|
||||||
|
var state = await index.DescribeAsync(Model, CancellationToken.None);
|
||||||
|
Assert.Equal(["b"], state.ContentHashBySlug.Keys);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ModelsAreIsolatedNamespaces()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
await index.UpsertAsync([new Embedding("m1", "a", "h1", Unit(1, 0, 0))], CancellationToken.None);
|
||||||
|
await index.UpsertAsync([new Embedding("m2", "b", "h2", Unit(1, 0, 0))], CancellationToken.None);
|
||||||
|
Assert.Equal(["a"], (await index.QueryAsync("m1", Unit(1, 0, 0), 10, CancellationToken.None)).Select(h => h.Slug));
|
||||||
|
Assert.Equal(["b"], (await index.QueryAsync("m2", Unit(1, 0, 0), 10, CancellationToken.None)).Select(h => h.Slug));
|
||||||
|
Assert.Equal(1, (await index.DescribeAsync("m1", CancellationToken.None)).Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SurvivesFlushAndReopen()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
await index.UpsertAsync([new Embedding(Model, "a", "h1", Unit(1, 0, 0)), new Embedding(Model, "b", "h2", Unit(0, 1, 0))], CancellationToken.None);
|
||||||
|
await index.FlushAsync(CancellationToken.None);
|
||||||
|
var reopened = await ReopenAsync(index);
|
||||||
|
var state = await reopened.DescribeAsync(Model, CancellationToken.None);
|
||||||
|
Assert.Equal(2, state.Count);
|
||||||
|
var hits = await reopened.QueryAsync(Model, Unit(0, 1, 0), 1, CancellationToken.None);
|
||||||
|
Assert.Equal("b", hits[0].Slug);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task HealthReportsOk()
|
||||||
|
{
|
||||||
|
var index = await CreateIndexAsync();
|
||||||
|
var health = await index.CheckAsync(CancellationToken.None);
|
||||||
|
Assert.True(health.Ok, health.Detail);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
using Wiki.Core.Model;
|
||||||
|
using Wiki.Core.Parsing;
|
||||||
|
|
||||||
|
namespace Wiki.Core.Tests;
|
||||||
|
|
||||||
|
public class FrontmatterTests
|
||||||
|
{
|
||||||
|
private const string Note = """
|
||||||
|
---
|
||||||
|
title: UDM intercepts DNS
|
||||||
|
type: wiki
|
||||||
|
tags: [dns, udm, "networking, lan"]
|
||||||
|
aliases:
|
||||||
|
- udm dns
|
||||||
|
- 'split horizon'
|
||||||
|
sources: [raw/_processed/a.md, "live check 2026-09-10: dig @1.1.1.1, dig @10.2.40.99"]
|
||||||
|
related: [ceph-osd, k8s-gateway-in-cluster-dns]
|
||||||
|
updated: 2026-09-16
|
||||||
|
empty:
|
||||||
|
---
|
||||||
|
|
||||||
|
# UDM intercepts DNS
|
||||||
|
|
||||||
|
Body line one.
|
||||||
|
""";
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ParsesScalarsFlowListsBlockListsAndQuotes()
|
||||||
|
{
|
||||||
|
var (fields, body, start) = Frontmatter.Parse(Note);
|
||||||
|
Assert.Equal("UDM intercepts DNS", fields["title"].Scalar);
|
||||||
|
Assert.Equal(["dns", "udm", "networking, lan"], fields["tags"].List!);
|
||||||
|
Assert.Equal(["udm dns", "split horizon"], fields["aliases"].List!);
|
||||||
|
Assert.Equal(["raw/_processed/a.md", "live check 2026-09-10: dig @1.1.1.1, dig @10.2.40.99"], fields["sources"].List!);
|
||||||
|
Assert.Equal(string.Empty, fields["empty"].Scalar);
|
||||||
|
Assert.Equal(13, start);
|
||||||
|
Assert.StartsWith("\n# UDM intercepts DNS", body);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void NoFrontmatterIsWholeBody()
|
||||||
|
{
|
||||||
|
var (fields, body, start) = Frontmatter.Parse("# Just a body\n");
|
||||||
|
Assert.Empty(fields);
|
||||||
|
Assert.Equal("# Just a body\n", body);
|
||||||
|
Assert.Equal(1, start);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void UnterminatedFrontmatterIsWholeBody()
|
||||||
|
{
|
||||||
|
var (fields, body, _) = Frontmatter.Parse("---\ntitle: x\nno end");
|
||||||
|
Assert.Empty(fields);
|
||||||
|
Assert.Equal("---\ntitle: x\nno end", body);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class DocumentParserTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void LiftsVaultFields()
|
||||||
|
{
|
||||||
|
var reference = new DocumentRef(DocumentKind.Wiki, "udm-dns", "wiki/udm-dns.md", "abc");
|
||||||
|
var doc = DocumentParser.Parse(reference, FrontmatterTests_Note.Text);
|
||||||
|
Assert.Equal("UDM intercepts DNS", doc.Title);
|
||||||
|
Assert.Equal("wiki", doc.Type);
|
||||||
|
Assert.Equal(new DateOnly(2026, 9, 16), doc.Updated);
|
||||||
|
Assert.Equal(["ceph-osd", "k8s-gateway-in-cluster-dns"], doc.Related);
|
||||||
|
Assert.Equal(2, doc.Sources.Count);
|
||||||
|
Assert.Equal(reference, doc.Ref);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void TitleFallsBackToSlug()
|
||||||
|
{
|
||||||
|
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Raw, "x", "raw/x.md", "h"), "no frontmatter");
|
||||||
|
Assert.Equal("x", doc.Title);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("wiki/a-b.md", DocumentKind.Wiki, "a-b")]
|
||||||
|
[InlineData("raw/2026-09-23-x.md", DocumentKind.Raw, "2026-09-23-x")]
|
||||||
|
[InlineData("raw/_processed/x.md", DocumentKind.RawProcessed, "x")]
|
||||||
|
[InlineData("output/2026-09-23-y.md", DocumentKind.Output, "2026-09-23-y")]
|
||||||
|
public void ClassifiesVaultPaths(string path, DocumentKind kind, string slug)
|
||||||
|
{
|
||||||
|
Assert.Equal((kind, slug), DocumentParser.Classify(path));
|
||||||
|
Assert.Equal(path, DocumentParser.PathFor(kind, slug));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Theory]
|
||||||
|
[InlineData("wiki/_index.md")]
|
||||||
|
[InlineData("bin/wiki_rank.py")]
|
||||||
|
[InlineData("Untitled.md")]
|
||||||
|
[InlineData("_archive/old.md")]
|
||||||
|
[InlineData("wiki/sub/x.md")]
|
||||||
|
public void IgnoresNonDocuments(string path) => Assert.Null(DocumentParser.Classify(path));
|
||||||
|
}
|
||||||
|
|
||||||
|
public class WikiLinksTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void ExtractsBodyLinksOutsideCodeAndRelated()
|
||||||
|
{
|
||||||
|
var text = "---\ntitle: T\nrelated: [rel-one, not a slug]\n---\n\nSee [[target-a]] and [[target-b|alias]] and [[target-c#heading]].\n`[[in-code]]`\n```\n[[fenced]]\n```\n[[ -f \"$X\" ]] and [[396,{{value|int}}]]\n";
|
||||||
|
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "src", "wiki/src.md", "h"), text);
|
||||||
|
var links = WikiLinks.Extract(doc);
|
||||||
|
Assert.Equal(
|
||||||
|
[("rel-one", LinkKind.Related, (int?)null), ("target-a", LinkKind.WikiLink, 6), ("target-b", LinkKind.WikiLink, 6), ("target-c", LinkKind.WikiLink, 6)],
|
||||||
|
links.Select(l => (l.ToSlug, l.Kind, l.Line)).ToArray());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ParagraphChunkerTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void SplitsOnBlankLinesHeadingsAndFences()
|
||||||
|
{
|
||||||
|
var text = "---\ntitle: UDM\n---\n\n# UDM intercepts DNS\n\nThe UDM answers every query.\n\n## Details\n\nEven dig.\nSecond line.\n\n```\ndig @10.2.40.99\n```\n";
|
||||||
|
var doc = DocumentParser.Parse(new DocumentRef(DocumentKind.Wiki, "udm", "wiki/udm.md", "h"), text);
|
||||||
|
var ps = new ParagraphChunker().Split(doc);
|
||||||
|
Assert.Equal(3, ps.Count);
|
||||||
|
Assert.Equal("udm#p1", ps[0].Id);
|
||||||
|
Assert.Equal("UDM intercepts DNS", ps[0].Heading);
|
||||||
|
Assert.Equal("Details", ps[1].Heading);
|
||||||
|
Assert.Contains("Second line", ps[1].Text);
|
||||||
|
Assert.StartsWith("```", ps[2].Text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static class FrontmatterTests_Note
|
||||||
|
{
|
||||||
|
public const string Text = """
|
||||||
|
---
|
||||||
|
title: UDM intercepts DNS
|
||||||
|
type: wiki
|
||||||
|
tags: [dns, udm]
|
||||||
|
sources: [raw/_processed/a.md, "live check: a, b"]
|
||||||
|
related: [ceph-osd, k8s-gateway-in-cluster-dns]
|
||||||
|
updated: 2026-09-16
|
||||||
|
---
|
||||||
|
|
||||||
|
# UDM intercepts DNS
|
||||||
|
""";
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user