feat: schedules move to cluster CronJobs #18

Merged
k.renner merged 1 commits from feat/external-schedules into main 2026-10-26 06:42:52 +00:00
8 changed files with 162 additions and 303 deletions
+17 -1
View File
@@ -1,9 +1,10 @@
using Wiki.Application.Encyclopedia;
using Wiki.Application.Status;
using Wiki.Application.Sync;
namespace Wiki.Host.Api;
/// <summary>/api/v1: the read API grows here phase by phase. Phase 1 exposes status and a manual sync.</summary>
/// <summary>/api/v1: status, and the sync and encyclopedia runs the cluster's CronJobs trigger.</summary>
public static class ApiV1
{
public static void MapApiV1(this WebApplication app)
@@ -25,5 +26,20 @@ public static class ApiV1
? Results.Json(new { error = sync.LastError }, statusCode: StatusCodes.Status502BadGateway)
: Results.Ok(summary);
}).RequireAuthorization(Auth.Policies.Write);
// Syncs first so the run plans from the current notes. The run outlives a caller that hangs up;
// only shutdown stops it. A run the text model stopped is a failure for the caller.
v1.MapPost("/encyclopedia/run", async (IServiceProvider services, SyncCoordinator sync, IHostApplicationLifetime lifetime) =>
{
var encyclopedia = services.GetService<EncyclopediaCoordinator>();
if (encyclopedia is null)
return Results.Json(new { error = "encyclopedia disabled" }, statusCode: StatusCodes.Status503ServiceUnavailable);
var ct = lifetime.ApplicationStopping;
await sync.RunAsync(force: false, ct);
var summary = await encyclopedia.RunAsync(ct);
return summary is null ? Results.Json(new { error = encyclopedia.LastError }, statusCode: StatusCodes.Status502BadGateway)
: summary.StoppedByModel ? Results.Json(summary, statusCode: StatusCodes.Status502BadGateway)
: Results.Ok(summary);
}).RequireAuthorization(Auth.Policies.Write);
}
}
@@ -1,81 +0,0 @@
using Wiki.Application.Encyclopedia;
using Wiki.Application.Sync;
namespace Wiki.Host.Composition;
/// <summary>
/// Reconciles the encyclopedia once a night at <see cref="EncyclopediaOptions.RunAt"/> in <see cref="EncyclopediaOptions.TimeZone"/>,
/// after a sync so the catalog is current. Never runs on startup. Nothing a run or the schedule throws stops the loop or the host.
/// </summary>
public sealed class EncyclopediaBackgroundService(
SyncCoordinator sync,
EncyclopediaCoordinator encyclopedia,
EncyclopediaOptions options,
TimeProvider clock,
ILogger<EncyclopediaBackgroundService> log) : BackgroundService
{
private static readonly TimeSpan ErrorBackoff = TimeSpan.FromMinutes(5);
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
TimeZoneInfo? zone = null;
while (!stoppingToken.IsCancellationRequested)
{
try
{
zone ??= ResolveZone();
var now = clock.GetUtcNow();
var target = NextRun(now, zone, options.RunAt);
log.LogInformation("next encyclopedia run at {Target:u}", target);
await Task.Delay(target - now, clock, stoppingToken);
// A timer may fire early; the run must not start before its target, or the next target would be the same one.
while ((now = clock.GetUtcNow()) < target)
await Task.Delay(target - now, clock, stoppingToken);
await sync.RunAsync(force: false, stoppingToken);
await encyclopedia.RunAsync(stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception e)
{
log.LogError(e, "encyclopedia schedule failed; scheduling the next run after a {Backoff} pause", ErrorBackoff);
try
{
await Task.Delay(ErrorBackoff, clock, stoppingToken);
}
catch (OperationCanceledException)
{
break;
}
}
}
}
/// <summary>The first <paramref name="runAt"/> in <paramref name="zone"/> strictly after <paramref name="now"/>; a time skipped by a DST change runs an hour later.</summary>
public static DateTimeOffset NextRun(DateTimeOffset now, TimeZoneInfo zone, TimeOnly runAt)
{
var today = TimeZoneInfo.ConvertTime(now, zone).Date;
for (var days = 0; ; days++)
{
var local = DateTime.SpecifyKind(today.AddDays(days) + runAt.ToTimeSpan(), DateTimeKind.Unspecified);
if (zone.IsInvalidTime(local)) local = local.AddHours(1);
var candidate = new DateTimeOffset(TimeZoneInfo.ConvertTimeToUtc(local, zone), TimeSpan.Zero);
if (candidate > now) return candidate;
}
}
private TimeZoneInfo ResolveZone()
{
try
{
return TimeZoneInfo.FindSystemTimeZoneById(options.TimeZone);
}
catch (Exception e) when (e is TimeZoneNotFoundException or InvalidTimeZoneException)
{
log.LogError(e, "time zone '{TimeZone}' cannot be resolved; scheduling in UTC", options.TimeZone);
return TimeZoneInfo.Utc;
}
}
}
@@ -2,8 +2,8 @@ using Wiki.Application.Sync;
namespace Wiki.Host.Composition;
/// <summary>Syncs at start, on every webhook trigger, and every interval; retries sooner after a failure.</summary>
public sealed class SyncBackgroundService(SyncCoordinator coordinator, WikiOptions options, ILogger<SyncBackgroundService> log) : BackgroundService
/// <summary>Syncs at start and on every webhook trigger. The schedule lives outside: a CronJob calls POST /api/v1/sync.</summary>
public sealed class SyncBackgroundService(SyncCoordinator coordinator, ILogger<SyncBackgroundService> log) : BackgroundService
{
private readonly SemaphoreSlim _wake = new(0);
@@ -13,13 +13,11 @@ public sealed class SyncBackgroundService(SyncCoordinator coordinator, WikiOptio
{
while (!stoppingToken.IsCancellationRequested)
{
var summary = await coordinator.RunAsync(force: false, stoppingToken);
var wait = summary is null ? options.Sync.RetrySeconds : options.Sync.IntervalSeconds;
if (summary is null)
log.LogWarning("sync failed; retrying in {Seconds}s", wait);
if (await coordinator.RunAsync(force: false, stoppingToken) is null)
log.LogWarning("sync failed; the next trigger retries");
try
{
await _wake.WaitAsync(TimeSpan.FromSeconds(wait), stoppingToken);
await _wake.WaitAsync(stoppingToken);
}
catch (OperationCanceledException)
{
+1 -12
View File
@@ -8,7 +8,6 @@ public sealed class WikiOptions
public JudgesOptions Judges { get; set; } = new();
public AnswerModelsOptions AnswerModels { get; set; } = new();
public IndexOptions Index { get; set; } = new();
public SyncScheduleOptions Sync { get; set; } = new();
public EncyclopediaOptions Encyclopedia { get; set; } = new();
}
@@ -77,23 +76,13 @@ public sealed class FileIndexStoreOptions
public string Directory { get; set; } = "/data";
}
public sealed class SyncScheduleOptions
{
/// <summary>Reconcile interval; the webhook is the fast path.</summary>
public int IntervalSeconds { get; set; } = 1800;
public int RetrySeconds { get; set; } = 60;
}
/// <summary>The nightly encyclopedia run. Off unless enabled; the API key is read from the answer model named by <see cref="AnswerModel"/>.</summary>
/// <summary>The encyclopedia run, triggered by POST /api/v1/encyclopedia/run. Off unless enabled; the API key is read from the answer model named by <see cref="AnswerModel"/>.</summary>
public sealed class EncyclopediaOptions
{
public bool Enabled { get; set; } = false;
/// <summary>The answer-model entry whose Url and key (`Wiki:AnswerModels:Models:{AnswerModel}:ApiKey`) the text model uses.</summary>
public string AnswerModel { get; set; } = "claude-sonnet";
public string Model { get; set; } = "claude-sonnet-5";
/// <summary>Local time in <see cref="TimeZone"/>; after the 04:30 distill.</summary>
public TimeOnly RunAt { get; set; } = new(6, 0);
public string TimeZone { get; set; } = "Europe/Berlin";
public int ArticleMaxTokens { get; set; } = 16000;
public int PlanMaxTokens { get; set; } = 32000;
public int MaxArticlesPerRun { get; set; } = 10;
+1 -2
View File
@@ -95,7 +95,7 @@ builder.Services.AddSingleton<IAnswerModel>(sp => sp.GetRequiredKeyedService<IAn
builder.Services.AddSingleton<IReadOnlyList<IFactDrafter>>(sp =>
wiki.AnswerModels.Models.Keys.Select(sp.GetRequiredKeyedService<IFactDrafter>).ToList());
builder.Services.AddSingleton<IFactDrafter>(sp => sp.GetRequiredKeyedService<IFactDrafter>(wiki.AnswerModels.Active));
// ---- encyclopedia: the text model and the nightly run exist only when enabled ------------
// ---- encyclopedia: the text model and the run exist only when enabled --------------------
var encyclopedia = wiki.Encyclopedia;
builder.Services.AddSingleton(encyclopedia);
string? encyclopediaConfigWarning = null;
@@ -120,7 +120,6 @@ if (encyclopedia.Enabled)
builder.Services.AddSingleton<Wiki.Application.Encyclopedia.IArticleWriter, Wiki.Application.Encyclopedia.WriteArticleUseCase>();
builder.Services.AddSingleton<Wiki.Application.Encyclopedia.IEncyclopediaRun, Wiki.Application.Encyclopedia.EncyclopediaReconciler>();
builder.Services.AddSingleton<Wiki.Application.Encyclopedia.EncyclopediaCoordinator>();
builder.Services.AddHostedService<EncyclopediaBackgroundService>();
}
// One adapter may serve several ports; list it once.
builder.Services.AddSingleton<IReadOnlyList<IAdapterHealth>>(sp =>
-1
View File
@@ -31,7 +31,6 @@
}
},
"Index": { "Active": "file", "File": { "Directory": "/data" } },
"Sync": { "IntervalSeconds": 1800, "RetrySeconds": 60 },
"Encyclopedia": { "Enabled": false }
},
"Auth": {
@@ -1,199 +0,0 @@
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Time.Testing;
using Wiki.Adapters.FileIndex;
using Wiki.Application.Encyclopedia;
using Wiki.Application.Sync;
using Wiki.Core.Ports;
using Wiki.Host.Composition;
namespace Wiki.Host.Tests;
public sealed class EncyclopediaBackgroundServiceTests : IAsyncLifetime
{
private static readonly TimeZoneInfo Berlin = TimeZoneInfo.FindSystemTimeZoneById("Europe/Berlin");
/// <summary>Signals each time something arms a timer, so tests move the clock only once the loop is waiting.</summary>
private sealed class SignalingClock : FakeTimeProvider
{
public SemaphoreSlim Armed { get; } = new(0);
public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
{
var timer = base.CreateTimer(callback, state, dueTime, period);
Armed.Release();
return timer;
}
}
private sealed class RecordingRun(IDocumentCatalog catalog) : IEncyclopediaRun
{
public List<string?> CatalogCommits { get; } = [];
public SemaphoreSlim Ran { get; } = new(0);
public Queue<Exception> Failures { get; } = new();
public async Task<ReconcileSummary> RunAsync(CancellationToken ct)
{
CatalogCommits.Add((await catalog.GetStateAsync(ct)).Commit);
Ran.Release();
if (Failures.TryDequeue(out var failure)) throw failure;
return new ReconcileSummary(0, 0, 0, 0, 0, false, null, null);
}
}
private readonly FakeStore _store = new();
private readonly FileDocumentCatalog _catalog;
private static readonly TimeSpan Patience = TimeSpan.FromSeconds(10);
private readonly SignalingClock _clock = new();
private readonly RecordingRun _run;
private readonly EncyclopediaCoordinator _coordinator;
private readonly SyncCoordinator _sync;
private readonly string _directory = Path.Combine(Path.GetTempPath(), "wiki-encyclopedia-tests", Guid.NewGuid().ToString("N"));
private EncyclopediaBackgroundService? _service;
public EncyclopediaBackgroundServiceTests()
{
var files = Options.Create(new FileIndexOptions { Directory = _directory });
_catalog = new FileDocumentCatalog(files);
var useCase = new SyncUseCase(_store, _catalog, new FileVectorIndex(files), [new FakeEmbeddings("m")], _clock, NullLogger<SyncUseCase>.Instance);
_sync = new SyncCoordinator(useCase, _clock, NullLogger<SyncCoordinator>.Instance);
_run = new RecordingRun(_catalog);
_coordinator = new EncyclopediaCoordinator(_run, _clock, NullLogger<EncyclopediaCoordinator>.Instance);
}
public Task InitializeAsync() => Task.CompletedTask;
public async Task DisposeAsync()
{
if (_service is not null) await _service.StopAsync(CancellationToken.None);
if (Directory.Exists(_directory)) Directory.Delete(_directory, recursive: true);
}
private static DateTimeOffset Berlin_(int day, int hour, int minute) =>
new(TimeZoneInfo.ConvertTimeToUtc(new DateTime(2026, 10, day, hour, minute, 0, DateTimeKind.Unspecified), Berlin), TimeSpan.Zero);
private async Task StartAtAsync(DateTimeOffset start, string timeZone = "Europe/Berlin")
{
_clock.SetUtcNow(start);
_service = new EncyclopediaBackgroundService(_sync, _coordinator, new EncyclopediaOptions { TimeZone = timeZone },
_clock, NullLogger<EncyclopediaBackgroundService>.Instance);
await _service.StartAsync(CancellationToken.None);
await WaitArmedAsync();
}
/// <summary>Moves the clock without firing the pending timer; the loop is waiting, so nothing can run.</summary>
private void AdvanceTo(DateTimeOffset target) => _clock.SetUtcNow(target);
/// <summary>Moves the clock to a run time, waits for the run, then for the loop to arm its next timer.</summary>
private async Task RunAtAsync(DateTimeOffset target)
{
_clock.SetUtcNow(target);
Assert.True(await _run.Ran.WaitAsync(Patience), "the run did not start");
await WaitArmedAsync();
}
private async Task WaitArmedAsync() => Assert.True(await _clock.Armed.WaitAsync(Patience), "the service did not arm a timer");
[Fact]
public async Task Service_RunsOncePerDayAtRunAt()
{
await StartAtAsync(Berlin_(24, 5, 0));
Assert.Empty(_run.CatalogCommits);
await RunAtAsync(Berlin_(24, 6, 0));
Assert.Single(_run.CatalogCommits);
AdvanceTo(Berlin_(25, 5, 59));
Assert.Single(_run.CatalogCommits);
await RunAtAsync(Berlin_(25, 6, 0));
Assert.Equal(2, _run.CatalogCommits.Count);
}
[Fact]
public async Task Service_DoesNotRunOnStartup()
{
await StartAtAsync(Berlin_(24, 7, 0));
Assert.Empty(_run.CatalogCommits);
AdvanceTo(Berlin_(25, 5, 59));
Assert.Empty(_run.CatalogCommits);
await RunAtAsync(Berlin_(25, 6, 0));
Assert.Single(_run.CatalogCommits);
}
[Fact]
public async Task Service_SyncsBeforeRun()
{
_store.Files["wiki/a.md"] = "---\ntitle: A\ntype: wiki\n---\n\nBody.\n";
await _sync.RunAsync(force: true, CancellationToken.None);
var first = (await _catalog.GetStateAsync(CancellationToken.None)).Commit;
await StartAtAsync(Berlin_(24, 5, 0));
_store.Files["wiki/b.md"] = "---\ntitle: B\ntype: wiki\n---\n\nBody.\n";
var head = (await _store.GetHeadAsync(CancellationToken.None)).Commit;
await RunAtAsync(Berlin_(24, 6, 0));
var seen = Assert.Single(_run.CatalogCommits);
Assert.NotEqual(first, seen);
Assert.Equal(head, seen);
}
[Fact]
public async Task Service_ReconcilerThrows_KeepsRunning()
{
_run.Failures.Enqueue(new JsonException("bad reply"));
await StartAtAsync(Berlin_(24, 5, 0));
await RunAtAsync(Berlin_(24, 6, 0));
Assert.Single(_run.CatalogCommits);
Assert.Equal("bad reply", _coordinator.LastError);
Assert.False(_service!.ExecuteTask!.IsCompleted);
await RunAtAsync(Berlin_(25, 6, 0));
Assert.Equal(2, _run.CatalogCommits.Count);
Assert.Null(_coordinator.LastError);
}
[Fact]
public async Task Service_UnknownTimeZone_FallsBackToUtc()
{
await StartAtAsync(new DateTimeOffset(2026, 10, 24, 5, 0, 0, TimeSpan.Zero), "Mars/Olympus");
await RunAtAsync(new DateTimeOffset(2026, 10, 24, 6, 0, 0, TimeSpan.Zero));
Assert.Single(_run.CatalogCommits);
}
[Fact]
public void NextRun_SkippedByDstGap_RunsAnHourLater()
{
// 2026-03-29 02:30 does not exist in Berlin
var next = EncyclopediaBackgroundService.NextRun(new DateTimeOffset(2026, 3, 28, 12, 0, 0, TimeSpan.Zero), Berlin, new TimeOnly(2, 30));
Assert.Equal(new DateTimeOffset(2026, 3, 29, 1, 30, 0, TimeSpan.Zero), next);
}
[Fact]
public void Host_EncyclopediaDisabled_NoTextModelRegistered()
{
using var factory = new WikiHostFactory();
Assert.Null(factory.Services.GetService<ITextModel>());
Assert.Null(factory.Services.GetService<EncyclopediaCoordinator>());
}
[Fact]
public void Host_EncyclopediaEnabled_TextModelInAdapterHealth()
{
using var factory = new WikiHostFactory().WithWebHostBuilder(b => b.UseSetting("Wiki:Encyclopedia:Enabled", "true"));
var adapters = factory.Services.GetRequiredService<IReadOnlyList<IAdapterHealth>>();
Assert.Contains(adapters, a => a.Name.StartsWith("anthropic-text:", StringComparison.Ordinal));
Assert.NotNull(factory.Services.GetRequiredService<EncyclopediaCoordinator>());
}
}
@@ -0,0 +1,138 @@
using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Wiki.Application.Encyclopedia;
using Wiki.Core.Ports;
namespace Wiki.Host.Tests;
public sealed class EncyclopediaEndpointTests
{
private sealed class RecordingRun(IDocumentCatalog catalog) : IEncyclopediaRun
{
public List<string?> CatalogCommits { get; } = [];
public Exception? Failure { get; set; }
public bool StoppedByModel { get; set; }
public async Task<ReconcileSummary> RunAsync(CancellationToken ct)
{
CatalogCommits.Add((await catalog.GetStateAsync(ct)).Commit);
if (Failure is not null) throw Failure;
return new ReconcileSummary(3, 0, 0, 0, 0, StoppedByModel, null, null);
}
}
private static (WebApplicationFactory<Program> Host, WikiHostFactory Base, Func<RecordingRun> Run) Enabled()
{
var factory = new WikiHostFactory();
var host = factory.WithWebHostBuilder(b =>
{
b.UseSetting("Wiki:Encyclopedia:Enabled", "true");
b.ConfigureTestServices(s =>
{
s.RemoveAll<IEncyclopediaRun>();
s.AddSingleton(sp => new RecordingRun(sp.GetRequiredService<IDocumentCatalog>()));
s.AddSingleton<IEncyclopediaRun>(sp => sp.GetRequiredService<RecordingRun>());
});
});
return (host, factory, () => host.Services.GetRequiredService<RecordingRun>());
}
private static HttpRequestMessage RunRequest(string? key = WikiHostFactory.ApiKey)
{
var req = new HttpRequestMessage(HttpMethod.Post, "/api/v1/encyclopedia/run");
if (key is not null) req.Headers.Add("X-Api-Key", key);
return req;
}
[Fact]
public void Host_EncyclopediaDisabled_NoTextModelRegistered()
{
using var factory = new WikiHostFactory();
Assert.Null(factory.Services.GetService<ITextModel>());
Assert.Null(factory.Services.GetService<EncyclopediaCoordinator>());
}
[Fact]
public void Host_EncyclopediaEnabled_TextModelInAdapterHealth()
{
using var factory = new WikiHostFactory().WithWebHostBuilder(b => b.UseSetting("Wiki:Encyclopedia:Enabled", "true"));
var adapters = factory.Services.GetRequiredService<IReadOnlyList<IAdapterHealth>>();
Assert.Contains(adapters, a => a.Name.StartsWith("anthropic-text:", StringComparison.Ordinal));
Assert.NotNull(factory.Services.GetRequiredService<EncyclopediaCoordinator>());
}
[Fact]
public async Task Run_Disabled_Is503()
{
using var factory = new WikiHostFactory();
var r = await factory.CreateClient().SendAsync(RunRequest());
Assert.Equal(HttpStatusCode.ServiceUnavailable, r.StatusCode);
}
[Fact]
public async Task Run_RequiresAnApiKey()
{
var (host, _, run) = Enabled();
using var _h = host;
var client = host.CreateClient();
Assert.Equal(HttpStatusCode.Unauthorized, (await client.SendAsync(RunRequest(key: null))).StatusCode);
Assert.Equal(HttpStatusCode.Unauthorized, (await client.SendAsync(RunRequest("wrong"))).StatusCode);
Assert.Empty(run().CatalogCommits);
}
[Fact]
public async Task Run_SyncsFirstAndReturnsTheSummary()
{
var (host, factory, run) = Enabled();
using var _h = host;
var client = host.CreateClient();
factory.Store.Files["wiki/a.md"] = "---\ntitle: A\ntype: wiki\n---\n\nBody.\n";
var head = (await factory.Store.GetHeadAsync(CancellationToken.None)).Commit;
var r = await client.SendAsync(RunRequest());
Assert.Equal(HttpStatusCode.OK, r.StatusCode);
Assert.Equal(head, Assert.Single(run().CatalogCommits));
var body = await r.Content.ReadFromJsonAsync<JsonElement>();
Assert.Equal(3, body.GetProperty("planned").GetInt32());
}
[Fact]
public async Task Run_Failed_Is502WithTheError()
{
var (host, _, run) = Enabled();
using var _h = host;
var client = host.CreateClient();
run().Failure = new JsonException("bad reply");
var r = await client.SendAsync(RunRequest());
Assert.Equal(HttpStatusCode.BadGateway, r.StatusCode);
Assert.Equal("bad reply", (await r.Content.ReadFromJsonAsync<JsonElement>()).GetProperty("error").GetString());
}
[Fact]
public async Task Run_StoppedByModel_Is502WithTheSummary()
{
var (host, _, run) = Enabled();
using var _h = host;
var client = host.CreateClient();
run().StoppedByModel = true;
var r = await client.SendAsync(RunRequest());
Assert.Equal(HttpStatusCode.BadGateway, r.StatusCode);
Assert.True((await r.Content.ReadFromJsonAsync<JsonElement>()).GetProperty("stoppedByModel").GetBoolean());
}
}