From 2205df61d0468311a3042283970f6af0087f843b Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Thu, 6 Aug 2026 12:53:27 -0300 Subject: [PATCH 1/2] Bundle CDN fetcher: skip unchanged product folders via the shallow registry map --- .../ReleaseNotes/CdnChangelogFetcher.cs | 155 +++++++++--- .../ReleaseNotes/ChangelogRegistry.cs | 3 + .../ReleaseNotes/CdnChangelogFetcherTests.cs | 237 ++++++++++++++++-- 3 files changed, 337 insertions(+), 58 deletions(-) diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs index 08cda2c97f..2dbae858ca 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogFetcher.cs @@ -21,7 +21,13 @@ namespace Elastic.Documentation.Configuration.ReleaseNotes; /// /// Individual bundle files are cached locally keyed by {product}-{fileName}-{etag} so that /// repeated builds (and dev-server reloads) do not re-download unchanged content from the CDN. -/// The registry itself is always fetched (it's small and provides fresh ETags). +/// The per-product registry is normally fetched every run (it's small and provides fresh ETags), +/// with one opt-out: the scrubber maintains a shallow per-tree map at bundle/registry.json +/// mapping each product folder to an opaque change token. The map is fetched once per fetcher run; +/// when a product's token equals the token the local cache last saw, the cached registry is reused +/// and the per-product registry fetch is skipped entirely. Tokens are opaque — compared for string +/// equality only, never parsed — and a map that is absent (pre-cutover CDNs), unparseable, or +/// unreachable degrades to exactly the pre-map behavior: every product registry is fetched. /// /// /// Resilience follows the manifest's consistency model: a registry that cannot be fetched or parsed @@ -60,6 +66,13 @@ public sealed class CdnChangelogFetcher : IDisposable private readonly IFileSystem _fileSystem; private readonly ConcurrentDictionary _memoryCache = new(StringComparer.Ordinal); + /// + /// Shallow-map fetches memoized per base URI, so one run consults the CDN once no matter how many + /// products it fetches. The map is intentionally never cached to disk: it is the freshness signal + /// itself, and a stale copy would defeat its purpose. + /// + private readonly ConcurrentDictionary?>>> _shallowMaps = new(StringComparer.Ordinal); + /// /// Non-null only when a caller injects its own (tests): in that case we /// own a per-instance client and must dispose it. On the production path points @@ -108,22 +121,30 @@ public async Task> FetchAsync( } var registryUri = Combine(baseUri, [.. ChangelogKeys.BundleSegments(product), ChangelogKeys.RegistryFileName]); + var shallowToken = await TryGetShallowTokenAsync(baseUri, product, ctx).ConfigureAwait(false); - ChangelogRegistry? registry; - try - { - registry = await FetchRegistryAsync(registryUri, ctx).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - emitError($"Could not fetch changelog registry for product '{product}' from {registryUri}: {ex.Message}"); - return []; - } - + var registry = TryGetCachedRegistry(product, shallowToken); if (registry is null) { - emitError($"Changelog registry for product '{product}' at {registryUri} was empty or unparseable."); - return []; + try + { + _logger.LogInformation("Fetching changelog registry {RegistryUri}", registryUri); + var registryText = await FetchTextAsync(registryUri, ctx).ConfigureAwait(false); + registry = JsonSerializer.Deserialize(registryText, ChangelogRegistryJsonContext.Default.ChangelogRegistry); + if (registry is not null && shallowToken is not null) + WriteCachedText(RegistryCacheKey(product, shallowToken), registryText); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + emitError($"Could not fetch changelog registry for product '{product}' from {registryUri}: {ex.Message}"); + return []; + } + + if (registry is null) + { + emitError($"Changelog registry for product '{product}' at {registryUri} was empty or unparseable."); + return []; + } } if (registry.SchemaVersion > SupportedSchemaVersion) @@ -143,14 +164,73 @@ public async Task> FetchAsync( return _bundleLoader.LoadBundlesFromContent(contents, emitWarning); } - private async Task FetchRegistryAsync(Uri registryUri, Cancel ctx) + /// + /// The product's opaque change token from the tree's shallow map, or null when the map is + /// unavailable or does not list the product — in which case the caller fetches the per-product + /// registry exactly as it did before the map existed. + /// + private async Task TryGetShallowTokenAsync(Uri baseUri, string product, Cancel ctx) { - _logger.LogInformation("Fetching changelog registry {RegistryUri}", registryUri); - using var request = new HttpRequestMessage(HttpMethod.Get, registryUri); - using var response = await _httpClient.SendAsync(request, ctx).ConfigureAwait(false); - _ = response.EnsureSuccessStatusCode(); - await using var stream = await response.Content.ReadAsStreamAsync(ctx).ConfigureAwait(false); - return await JsonSerializer.DeserializeAsync(stream, ChangelogRegistryJsonContext.Default.ChangelogRegistry, ctx).ConfigureAwait(false); + var lazyMap = _shallowMaps.GetOrAdd( + baseUri.AbsoluteUri, + _ => new Lazy?>>(() => FetchShallowMapAsync(baseUri, ctx))); + var map = await lazyMap.Value.ConfigureAwait(false); + if (map is null || !map.TryGetValue(product, out var token)) + return null; + + // The token is opaque but becomes part of a local cache file name; anything that is not a + // plain path segment is ignored rather than joined into a path. + return ChangelogKeys.IsSafeFileName(token) ? token : null; + } + + /// + /// Fetches the tree's shallow map (bundle/registry.json) mapping each product folder to an + /// opaque change token. Every failure — absent on pre-cutover CDNs, unparseable, transport — degrades + /// to null so the run behaves exactly as it did before the map existed. + /// + private async Task?> FetchShallowMapAsync(Uri baseUri, Cancel ctx) + { + var mapUri = Combine(baseUri, ["bundle", ChangelogKeys.RegistryFileName]); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, mapUri); + using var response = await _httpClient.SendAsync(request, ctx).ConfigureAwait(false); + _ = response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(ctx).ConfigureAwait(false); + return await JsonSerializer.DeserializeAsync(stream, ChangelogRegistryJsonContext.Default.DictionaryStringString, ctx).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogDebug("Shallow changelog map at {MapUri} is unavailable ({Message}); fetching every product registry as usual", mapUri, ex.Message); + return null; + } + } + + /// + /// The parsed registry from the token-keyed local cache, or null when there is no shallow token + /// for the product, no cached copy for that token, or the cached copy no longer parses — every + /// miss falls back to a normal registry fetch. + /// + private ChangelogRegistry? TryGetCachedRegistry(string product, string? shallowToken) + { + if (shallowToken is null) + return null; + + var cached = TryGetCachedText(RegistryCacheKey(product, shallowToken)); + if (cached is null) + return null; + + try + { + var registry = JsonSerializer.Deserialize(cached, ChangelogRegistryJsonContext.Default.ChangelogRegistry); + if (registry is not null) + _logger.LogInformation("Changelog folder 'bundle/{Product}' is unchanged per the shallow map; using the cached registry", product); + return registry; + } + catch (JsonException) + { + return null; + } } private async Task> DownloadBundlesAsync( @@ -257,12 +337,17 @@ private static Uri Combine(Uri baseUri, IReadOnlyList segments) return new Uri($"{basePath}/{suffix}"); } - private string? TryGetCachedBundle(string product, string fileName, string? etag) + private string? TryGetCachedBundle(string product, string fileName, string? etag) => + string.IsNullOrWhiteSpace(etag) ? null : TryGetCachedText(BundleCacheKey(product, fileName, etag)); + + private void WriteCachedBundle(string product, string fileName, string? etag, string content) { - if (string.IsNullOrWhiteSpace(etag)) - return null; + if (!string.IsNullOrWhiteSpace(etag)) + WriteCachedText(BundleCacheKey(product, fileName, etag), content); + } - var cacheKey = CacheKey(product, fileName, etag); + private string? TryGetCachedText(string cacheKey) + { if (_memoryCache.TryGetValue(cacheKey, out var cached)) return cached; @@ -278,17 +363,13 @@ private static Uri Combine(Uri baseUri, IReadOnlyList segments) } catch (Exception e) { - _logger.LogError(e, "Failed to read cached changelog bundle {CachePath}", cachePath); + _logger.LogError(e, "Failed to read cached changelog file {CachePath}", cachePath); return null; } } - private void WriteCachedBundle(string product, string fileName, string? etag, string content) + private void WriteCachedText(string cacheKey, string content) { - if (string.IsNullOrWhiteSpace(etag)) - return; - - var cacheKey = CacheKey(product, fileName, etag); _ = _memoryCache.TryAdd(cacheKey, content); var cachePath = CachePath(cacheKey); @@ -302,13 +383,21 @@ private void WriteCachedBundle(string product, string fileName, string? etag, st } catch (Exception e) { - _logger.LogError(e, "Failed to write cached changelog bundle {CachePath}", cachePath); + _logger.LogError(e, "Failed to write cached changelog file {CachePath}", cachePath); } } - private static string CacheKey(string product, string fileName, string etag) => + private static string BundleCacheKey(string product, string fileName, string etag) => $"changelog-{product}-{fileName}-{etag}"; + /// + /// Registry cache entries embed the shallow token in the key: a token mismatch is simply a cache + /// miss under the new key, which re-fetches and records the fresh registry alongside it — the same + /// convention the ETag-keyed bundle cache follows. + /// + private static string RegistryCacheKey(string product, string token) => + $"registry-{product}-{token}"; + private static string CachePath(string cacheKey) => Path.Join(Paths.ApplicationData.FullName, "changelog-bundles", cacheKey); diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs index 72dfc3c837..d7f15e0189 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs @@ -43,7 +43,10 @@ public sealed record ChangelogRegistryBundle public string? ETag { get; init; } } +// Dictionary is the shallow per-tree map (bundle/registry.json): folder → opaque +// change token, maintained by the scrubber's ShallowRegistryReconciler. [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower)] [JsonSerializable(typeof(ChangelogRegistry))] [JsonSerializable(typeof(ChangelogRegistryBundle))] +[JsonSerializable(typeof(Dictionary))] internal sealed partial class ChangelogRegistryJsonContext : JsonSerializerContext; diff --git a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogFetcherTests.cs b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogFetcherTests.cs index b87cb5ba64..34d4523ca4 100644 --- a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogFetcherTests.cs +++ b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogFetcherTests.cs @@ -28,6 +28,9 @@ public class CdnChangelogFetcherTests private static readonly Uri BaseUri = new("https://cdn.example"); + /// The shallow per-tree map probed once per run before any per-product registry fetch. + private const string ShallowMapPath = "/bundle/registry.json"; + private static CdnChangelogFetcher CreateFetcher(StubHandler handler) => new(NullLoggerFactory.Instance, new FileSystem(), handler); @@ -289,10 +292,13 @@ public async Task FetchAsync_InvalidProduct_EmitsErrorAndDoesNotHitCdn(string pr [Fact] public async Task FetchAsync_WithETag_UsesCachedBundleOnSecondCall() { - var handler = new StubHandler(req => - req.RequestUri!.AbsolutePath.EndsWith("/registry.json", StringComparison.Ordinal) - ? Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "abc123" } ] }""") - : Yaml(SampleBundle)); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => NotFound(), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => + Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "abc123" } ] }"""), + _ => Yaml(SampleBundle) + }); var (errors, warnings, emitError, emitWarning) = Diagnostics(); var fs = new MockFileSystem(); @@ -301,12 +307,12 @@ public async Task FetchAsync_WithETag_UsesCachedBundleOnSecondCall() // First call — should fetch from CDN var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); bundles.Should().ContainSingle(); - handler.CallCount.Should().Be(2, "registry + bundle"); + handler.CallCount.Should().Be(3, "shallow map probe + registry + bundle"); - // Second call — bundle should come from cache (only registry re-fetched) + // Second call — bundle should come from cache (only registry re-fetched; the map probe is memoized per run) var bundles2 = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); bundles2.Should().ContainSingle(); - handler.CallCount.Should().Be(3, "only registry fetched again; bundle served from memory cache"); + handler.CallCount.Should().Be(4, "only registry fetched again; bundle served from memory cache"); errors.Should().BeEmpty(); } @@ -337,27 +343,33 @@ public async Task FetchAsync_WithETag_ReadsCacheFromDisk() fs.Directory.CreateDirectory(Path.GetDirectoryName(cachePath)!); fs.File.WriteAllText(cachePath, SampleBundle); - var handler = new StubHandler(req => - req.RequestUri!.AbsolutePath.EndsWith("/registry.json", StringComparison.Ordinal) - ? Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "cached1" } ] }""") - : throw new InvalidOperationException("Should not fetch bundle from CDN")); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => NotFound(), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => + Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "cached1" } ] }"""), + _ => throw new InvalidOperationException("Should not fetch bundle from CDN") + }); var (errors, warnings, emitError, emitWarning) = Diagnostics(); using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, handler); var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); bundles.Should().ContainSingle(); - handler.CallCount.Should().Be(1, "only registry should be fetched; bundle served from disk"); + handler.CallCount.Should().Be(2, "only the map probe and registry should be fetched; bundle served from disk"); errors.Should().BeEmpty(); } [Fact] public async Task FetchAsync_NullETag_AlwaysFetchesFromCdn() { - var handler = new StubHandler(req => - req.RequestUri!.AbsolutePath.EndsWith("/registry.json", StringComparison.Ordinal) - ? Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": null } ] }""") - : Yaml(SampleBundle)); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => NotFound(), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => + Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": null } ] }"""), + _ => Yaml(SampleBundle) + }); var (errors, _, emitError, emitWarning) = Diagnostics(); var fs = new MockFileSystem(); @@ -365,11 +377,11 @@ public async Task FetchAsync_NullETag_AlwaysFetchesFromCdn() // First call _ = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); - handler.CallCount.Should().Be(2); + handler.CallCount.Should().Be(3, "map probe + registry + bundle"); - // Second call — no caching, so bundle is fetched again + // Second call — no caching, so bundle is fetched again (the map probe stays memoized) _ = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); - handler.CallCount.Should().Be(4, "without ETag, both registry and bundle are fetched each time"); + handler.CallCount.Should().Be(5, "without ETag, both registry and bundle are fetched each time"); errors.Should().BeEmpty(); } @@ -377,25 +389,200 @@ public async Task FetchAsync_NullETag_AlwaysFetchesFromCdn() public async Task FetchAsync_ChangedETag_FetchesNewBundle() { var etag = "v1"; - var handler = new StubHandler(req => - req.RequestUri!.AbsolutePath.EndsWith("/registry.json", StringComparison.Ordinal) - ? Json($$"""{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "{{etag}}" } ] }""") - : Yaml(SampleBundle)); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => NotFound(), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => + Json($$"""{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "{{etag}}" } ] }"""), + _ => Yaml(SampleBundle) + }); var (errors, _, emitError, emitWarning) = Diagnostics(); var fs = new MockFileSystem(); using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, handler); _ = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); - handler.CallCount.Should().Be(2); + handler.CallCount.Should().Be(3, "map probe + registry + bundle"); // Simulate a new etag by creating a new fetcher (in real usage the registry returns a different etag) etag = "v2"; _ = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); - handler.CallCount.Should().Be(4, "new ETag means cache miss, bundle re-downloaded"); + handler.CallCount.Should().Be(5, "new ETag means cache miss, bundle re-downloaded"); errors.Should().BeEmpty(); } + // language=json + private const string EsRegistryJson = + """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "abc123" } ] }"""; + + private static string CacheFilePath(string cacheKey) => + Path.Join(Paths.ApplicationData.FullName, "changelog-bundles", cacheKey); + + /// Seeds the disk cache as a previous run with shallow token would have left it. + private static MockFileSystem WarmCache(string token) + { + var fs = new MockFileSystem(); + fs.Directory.CreateDirectory(Path.GetDirectoryName(CacheFilePath("x"))!); + fs.File.WriteAllText(CacheFilePath($"registry-elasticsearch-{token}"), EsRegistryJson); + fs.File.WriteAllText(CacheFilePath("changelog-elasticsearch-9.3.0.yaml-abc123"), SampleBundle); + return fs; + } + + [Fact] + public async Task FetchAsync_ShallowMapAbsent_FetchesRegistryAndBundleAsBefore() + { + // Pre-cutover CDNs have no bundle/registry.json: a 404 must degrade to the full per-product flow. + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => NotFound(), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => Json(EsRegistryJson), + _ => Yaml(SampleBundle) + }); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, new MockFileSystem(), handler); + var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + bundles.Should().ContainSingle(); + handler.RequestedPaths.Should().Contain("/bundle/elasticsearch/registry.json"); + handler.RequestedPaths.Should().Contain("/bundle/elasticsearch/9.3.0.yaml"); + } + + [Fact] + public async Task FetchAsync_ShallowMapUnparseable_FetchesRegistryAndBundleAsBefore() + { + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => Json("{ not valid json"), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => Json(EsRegistryJson), + _ => Yaml(SampleBundle) + }); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, new MockFileSystem(), handler); + var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + bundles.Should().ContainSingle(); + handler.RequestedPaths.Should().Contain("/bundle/elasticsearch/registry.json"); + handler.RequestedPaths.Should().Contain("/bundle/elasticsearch/9.3.0.yaml"); + } + + [Fact] + public async Task FetchAsync_ShallowTokenMatchesWarmCache_MakesNoPerProductRequests() + { + var fs = WarmCache("tok1"); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath == ShallowMapPath + ? Json(/*lang=json,strict*/ """{ "elasticsearch": "tok1" }""") + : throw new InvalidOperationException($"Unexpected per-folder request: {req.RequestUri}")); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, handler); + var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + bundles.Should().ContainSingle(); + bundles[0].Entries.Should().ContainSingle().Which.Title.Should().Be("Sample enhancement"); + handler.RequestedPaths.Should().Equal(ShallowMapPath); + } + + [Fact] + public async Task FetchAsync_ShallowTokenMismatch_FetchesRegistryAndRecordsNewToken() + { + var fs = WarmCache("tok-old"); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => Json(/*lang=json,strict*/ """{ "elasticsearch": "tok-new" }"""), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => Json(EsRegistryJson), + _ => Yaml(SampleBundle) + }); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, handler); + var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + bundles.Should().ContainSingle(); + handler.RequestedPaths.Should().Contain("/bundle/elasticsearch/registry.json"); + fs.File.Exists(CacheFilePath("registry-elasticsearch-tok-new")) + .Should().BeTrue("the fresh registry should be recorded under the new token for the next run"); + } + + [Fact] + public async Task FetchAsync_ShallowTokenWithColdCache_FetchesAsUsualThenSkipsOnNextRun() + { + var fs = new MockFileSystem(); + var mapJson = /*lang=json,strict*/ """{ "elasticsearch": "tok1" }"""; + var coldHandler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => Json(mapJson), + var p when p.EndsWith("/registry.json", StringComparison.Ordinal) => Json(EsRegistryJson), + _ => Yaml(SampleBundle) + }); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + // Cold cache: the token alone cannot satisfy a skip, so the flow is identical to today. + using (var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, coldHandler)) + { + var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + bundles.Should().ContainSingle(); + coldHandler.RequestedPaths.Should().Contain("/bundle/elasticsearch/registry.json"); + } + + // Next run (new fetcher, same disk cache): the unchanged token skips every per-product request. + var warmHandler = new StubHandler(req => req.RequestUri!.AbsolutePath == ShallowMapPath + ? Json(mapJson) + : throw new InvalidOperationException($"Unexpected per-folder request: {req.RequestUri}")); + using (var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, warmHandler)) + { + var bundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + bundles.Should().ContainSingle(); + warmHandler.RequestedPaths.Should().Equal(ShallowMapPath); + } + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + } + + [Fact] + public async Task FetchAsync_ShallowMapPartialMatch_SkipsOnlyUnchangedFolders() + { + // One run over two products: elasticsearch has a warm cache and a matching token, kibana does + // not — only kibana's registry and bundle may hit the CDN, and the map is probed exactly once. + var fs = WarmCache("tok-es"); + var handler = new StubHandler(req => req.RequestUri!.AbsolutePath switch + { + ShallowMapPath => Json(/*lang=json,strict*/ """{ "elasticsearch": "tok-es", "kibana": "tok-kb" }"""), + "/bundle/kibana/registry.json" => + Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "kibana", "bundles": [ { "file": "9.3.0.yaml", "target": "9.3.0", "etag": "kb1" } ] }"""), + "/bundle/kibana/9.3.0.yaml" => Yaml(SampleBundle), + var p => throw new InvalidOperationException($"Unexpected per-folder request: {p}") + }); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + using var fetcher = new CdnChangelogFetcher(NullLoggerFactory.Instance, fs, handler); + var esBundles = await fetcher.FetchAsync(BaseUri, "elasticsearch", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + var kibanaBundles = await fetcher.FetchAsync(BaseUri, "kibana", version: null, emitError, emitWarning, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + esBundles.Should().ContainSingle(); + kibanaBundles.Should().ContainSingle(); + handler.RequestedPaths.Count(p => p == ShallowMapPath).Should().Be(1, "the map is fetched once per run"); + handler.RequestedPaths.Should().NotContain(p => p.StartsWith("/bundle/elasticsearch/", StringComparison.Ordinal)); + handler.RequestedPaths.Should().Contain("/bundle/kibana/registry.json"); + handler.RequestedPaths.Should().Contain("/bundle/kibana/9.3.0.yaml"); + fs.File.Exists(CacheFilePath("registry-kibana-tok-kb")) + .Should().BeTrue("kibana's registry should be recorded under its token for the next run"); + } + + private static HttpResponseMessage NotFound() => new(HttpStatusCode.NotFound); + private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK) { Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") }; From be6ddb6868e27866650489efec75e151e188b16d Mon Sep 17 00:00:00 2001 From: lcawl Date: Fri, 21 Aug 2026 10:55:50 -0500 Subject: [PATCH 2/2] Fix stale changelog registry --- docs/cli/changelog/cmd-bundle.md | 12 +- docs/cli/changelog/cmd-upload.md | 7 +- docs/data/release-notes/configure-ref.md | 2 +- docs/development/changelog-bundle-registry.md | 46 +++--- .../ReleaseNotes/CdnChangelogEntryFetcher.cs | 82 +++++++++- .../ReleaseNotes/ChangelogKeys.cs | 2 +- .../ReleaseNotes/ChangelogRegistry.cs | 2 +- .../docs-lambda-changelog-scrubber/README.md | 21 ++- .../Bundling/ChangelogBundlingService.cs | 110 ++++++++------ .../Bundling/ChangelogEntryMatcher.cs | 18 ++- .../Bundling/ChangelogPrIdentity.cs | 50 ++++++ .../Bundling/GitRangeEntryResolver.cs | 29 +--- .../BundleRegistryReconciler.cs | 23 ++- .../Scrubbing/ScrubberProcessor.cs | 28 ++-- .../Elastic.Changelog/Uploading/Registry.cs | 19 +-- .../Changelogs/BundleCdnSourcingTests.cs | 143 +++++++++++++++++- .../Changelogs/BundleChangelogsTests.cs | 62 ++++++++ .../Changelogs/BundleFilesFilterTests.cs | 10 +- .../Changelogs/CloudProfileFixtureTests.cs | 17 ++- .../BundleRegistryReconcilerTests.cs | 52 ++++++- .../Scrubbing/ScrubberProcessorTests.cs | 35 +++-- .../CdnChangelogEntryFetcherTests.cs | 36 +++++ 22 files changed, 604 insertions(+), 202 deletions(-) create mode 100644 src/services/Elastic.Changelog/Bundling/ChangelogPrIdentity.cs diff --git a/docs/cli/changelog/cmd-bundle.md b/docs/cli/changelog/cmd-bundle.md index 726c63025a..b54b27731f 100644 --- a/docs/cli/changelog/cmd-bundle.md +++ b/docs/cli/changelog/cmd-bundle.md @@ -43,13 +43,13 @@ Supply filter flags directly when you don't have a profile configured or need a Exactly one of the following filter flags is required: -- `--all` — include every changelog in the directory -- `--input-products` — match by product, target date, and lifecycle (e.g. `"elasticsearch * *"`) -- `--prs` — filter by PR URLs or a newline-delimited file of PR URLs -- `--issues` — filter by issue URLs or a newline-delimited file of issue URLs +- `--all` — include every changelog in the directory (local sourcing only; on the CDN this errors and asks for `--force-local`) +- `--input-products` — match by product, target date, and lifecycle (e.g. `"elasticsearch * *"`). On the CDN this also requires a PR, issue, or file identity; product-only CDN runs error and ask for `--force-local`. +- `--prs` — filter by PR URLs or a newline-delimited file of PR URLs. An entry matches when **leading filename digits** equal the PR number **or** YAML `prs:` contains it (the same join git-ref uses). Empty `prs:` is fine when the filename carries the PR (for example `12345.yaml` after public scrubbing). +- `--issues` — filter by issue URLs or a newline-delimited file of issue URLs (YAML `issues:` on the downloaded pool) - `--release-version` — fetch PR references from a GitHub release tag (e.g. `v9.2.0` or `latest`) - `--report` — filter by PRs referenced in a promotion report (URL or local file) -- `--files` — include specific changelog YAML paths, or a newline-delimited path list file +- `--files` — include specific changelog YAML paths, or a newline-delimited path list file. On the CDN this GETs those basenames only and does not read `registry.json`. - `--start-git-ref` + `--end-git-ref` — derive the PR list from a git commit range (see [Commit-range mode](#git-ref-mode)) `--force-local` is not a filter. It forces local entry sourcing for the run (equivalent to `bundle.use_local_changelogs: true` without editing config) and is allowed in both option-based and profile-based modes. @@ -408,7 +408,7 @@ In profile mode, pass the same path list as a positional argument: docs-builder changelog bundle serverless-release 2026-07-07 ./docs/temp/changelog_files.txt ``` -`--files` / path-list selection follows the standard entry-sourcing rules. When entries are sourced from the CDN (the default when `bundle.repo` resolves), the listed paths are matched to CDN pool entries by file name and do not need to exist locally — useful for private repositories whose entries exist only in S3 and whose public copies have PR/issue references scrubbed, so PR-based filters cannot match. With local sourcing (`--force-local`, `--directory`, or `bundle.use_local_changelogs`), the listed files are read from disk and must exist. In either mode, a listed entry that cannot be found fails the run, and `rules.bundle` still applies after selection. +`--files` / path-list selection follows the standard entry-sourcing rules. When entries are sourced from the CDN (the default when `bundle.repo` resolves), the listed paths are **GET by file name** and do not need to exist locally — the pool `registry.json` is not read. This is useful for private repositories whose entries exist only in S3. With local sourcing (`--force-local`, `--directory`, or `bundle.use_local_changelogs`), the listed files are read from disk and must exist. In either mode, a listed entry that cannot be found fails the run, and `rules.bundle` still applies after selection. ### Force local entry sourcing [changelog-bundle-force-local] diff --git a/docs/cli/changelog/cmd-upload.md b/docs/cli/changelog/cmd-upload.md index 2fb23b63b7..7778760ef4 100644 --- a/docs/cli/changelog/cmd-upload.md +++ b/docs/cli/changelog/cmd-upload.md @@ -103,10 +103,9 @@ s3://{bucket}/bundle/{product}/{filename} # --artifact-type bun Changelog entries are written once under the authoring org/repo/branch. A bundle that applies to multiple products is uploaded to multiple keys — one per product. The command writes YAML objects only — it never writes a `registry.json`. The public -`bundle/{product}/registry.json` manifests are produced exclusively by the scrubber Lambda, -reconciled from public bucket state on the S3 events each upload emits; the -`changelog/{org}/{repo}/{branch}/registry.json` pool manifests are legacy client-authored -objects that only older CLI versions still write. See +`bundle/{product}/registry.json` and `changelog/{org}/{repo}/{branch}/registry.json` +manifests are produced exclusively by the scrubber Lambda, reconciled from public +bucket state on the S3 events each upload emits. See [Changelog bundle registry](/development/changelog-bundle-registry.md). When several repositories publish bundles for the same shared product (for example `cloud-serverless`), use a `{repo}-{dateOrVersion}.yaml` bundle filename convention so they don't overwrite each other under `bundle/{product}/`. diff --git a/docs/data/release-notes/configure-ref.md b/docs/data/release-notes/configure-ref.md index 5e8add87d2..2ada9ccf0f 100644 --- a/docs/data/release-notes/configure-ref.md +++ b/docs/data/release-notes/configure-ref.md @@ -73,7 +73,7 @@ The authoring repo is resolved with the same precedence as `changelog upload`: ` Sourcing is decided per run: - **Local folder.** Used when `bundle.use_local_changelogs: true`, when `--force-local` is passed, when `--directory` is passed, or when the authoring repo cannot be resolved. The folder must contain the changelog files. -- **CDN (default when a repo resolves).** Used when the authoring repo resolves, local sourcing is not forced, and a CDN base URL is configured (`DOCS_BUILDER_CHANGELOG_CDN`, defaulting to the public distribution). The command fetches `changelog/{org}/{repo}/{branch}/registry.json` and the entries it lists, then applies the bundle's own product/PR/issue/file filters to the downloaded set. Path-list / `--files` filters match pool entries by file name, so the listed paths do not need to exist locally. +- **CDN (default when a repo resolves).** Used when the authoring repo resolves, local sourcing is not forced, and a CDN base URL is configured (`DOCS_BUILDER_CHANGELOG_CDN`, defaulting to the public distribution). `--prs`, `--issues`, `--report`, `--release-version`, a URL list, and git-ref fetch `changelog/{org}/{repo}/{branch}/registry.json` and the entries it lists, then apply the same PR join locally and on the CDN: **filename-derived PR numbers or YAML `prs:`**. Path-list / `--files` GETs those pool objects by file name and does not read the registry. CDN `--all` and product-only filters (no PR, issue, or file identity) error; pass `--force-local` to read the local folder. Local `--all` is unchanged. Use `--force-local` for uncommon ad hoc runs that need the local folder without editing `changelog.yml` — including path-list / `--files` runs that should read freshly authored files from disk instead of the CDN pool. diff --git a/docs/development/changelog-bundle-registry.md b/docs/development/changelog-bundle-registry.md index fa04e2ccf1..f9de2ef821 100644 --- a/docs/development/changelog-bundle-registry.md +++ b/docs/development/changelog-bundle-registry.md @@ -29,7 +29,7 @@ copies, no cross-repo file syncing. flowchart LR CI["Client CI
(docs-actions)"] -->|"changelog upload
(YAML objects only)"| Private["Private S3 bucket
bundle/{product}/*.yaml
changelog/{org}/{repo}/{branch}/*.yaml"] Private -->|"s3:ObjectCreated / ObjectRemoved
→ SQS"| Scrubber["Changelog scrubber
Lambda"] - Scrubber -->|"scrub + copy/delete,
then reconcile bundle registry.json
+ shallow maps from public listing"| Public["Public S3 bucket
+ CloudFront CDN
(incl. registry.json)"] + Scrubber -->|"scrub + copy/delete,
then reconcile bundle and pool registry.json
+ shallow maps from public listing"| Public["Public S3 bucket
+ CloudFront CDN
(incl. registry.json)"] Public -->|"reads via CDN"| Directive["{changelog} directive
(cdn: mode)"] ``` @@ -60,21 +60,18 @@ Both indexes share this schema, serialized with `snake_case` keys. ### Ownership per tree [ownership-per-tree] -The two trees part ways on who writes the manifest -(the [2026-08-10 update on elastic/docs-eng-team#688](https://github.com/elastic/docs-eng-team/issues/688) -narrowed reconciliation to the bundle tree): +The two trees share a producer and differ only in what each listing records: - **Bundle index** — `bundle/{product}/registry.json`, **public bucket only**, produced - exclusively by the scrubber Lambda's `BundleRegistryReconciler`. This is the manifest the - `{changelog}` directive and external CDN consumers enumerate, and the subject of the rest of - this page. -- **Changelog-entry index** — `changelog/{org}/{repo}/{branch}/registry.json`, a **legacy - client-authored pass-through**: the current `changelog upload` never writes one, but manifests - written by older CLI versions are still mirrored verbatim from the private bucket, because - [`changelog bundle` entry sourcing](#entry-sourcing) still enumerates a pool through its - manifest. It is *not* reconciled — its `producer` is null and its recorded `etag` is the old - pre-scrub private-object hash (consumers ignore it). It goes away entirely once release-note - discovery starts from PR lists (RFC [elastic/docs-eng-team#698](https://github.com/elastic/docs-eng-team/issues/698)). + exclusively by the scrubber Lambda's `BundleRegistryReconciler`. Each entry records a + `target` (version or date) so the `{changelog}` directive and external CDN consumers can + enumerate bundles. This is the subject of most of the rest of this page. +- **Changelog-entry index** — `changelog/{org}/{repo}/{branch}/registry.json`, **public + bucket only**, produced by the same reconciler from the public YAML listing. It is + **listing-only** (`target` is always null). `changelog bundle --prs` and git-ref still + download this listing first (the public CDN cannot `ListObjects`); `--files` / a path + list GETs named objects and does not read it. Upload never writes this file. Stale + leftover client JSON is healed on the next YAML or registry-key event. ```json { @@ -149,19 +146,19 @@ event's *type* — an event only means "this key may have changed": *current* content and PUT to public; 404 → conditionally delete the public copy. After the write, a HEAD re-validates that the private object still matches the snapshot the write was derived from, redoing the reconcile if a concurrent invocation raced it. -2. **Group reconcile** (`bundle/{product}/` keys only — the pool tree has none) — list the - group's public prefix (paginated), reuse entries whose recorded ETag still matches the - listing, GET and recompute the rest (amends always recomputed), and write the manifest back. +2. **Group reconcile** (`bundle/{product}/` and `changelog/{org}/{repo}/{branch}/` keys) — + list the group's public prefix (paginated), reuse entries whose recorded ETag still + matches the listing, GET and recompute bundle targets (pool listings skip the YAML GET + and record a null `target`), and write the manifest back. 3. **Shallow-map reconcile** — patch the touched tree's [folder→token map](#shallow-maps) from the same public listings. Within an SQS batch this work is coalesced: one object reconcile per distinct key, one group reconcile per distinct group, one shallow-map reconcile per touched tree. -Registry-key events split by tree. A **bundle** manifest is never copied or deleted — the event -only schedules the group reconcile, so client-authored JSON never reaches the tree consumers -enumerate. A **pool** manifest is mirrored verbatim (the -[legacy pass-through](#ownership-per-tree)). Any other `.json` key is skipped with a warning. +Registry-key events never copy or delete the JSON object — the event only schedules the +group reconcile, so client-authored JSON never reaches the tree consumers enumerate. Any +other `.json` key is skipped with a warning. ### Concurrency: optimistic, conditional writes @@ -235,8 +232,11 @@ The `changelog bundle` command aggregates individual changelog **entries**. It c entries from the local folder or fetch the **authoring pool's** published entries from the CDN (`changelog/{org}/{repo}/{branch}/registry.json` → `changelog/{org}/{repo}/{branch}/{file}`, via `CdnChangelogEntryFetcher`). The pool manifest it enumerates is the -[legacy client-authored index](#ownership-per-tree); this enumeration is what keeps the -pass-through alive until PR-list-driven discovery (RFC elastic/docs-eng-team#698) replaces it. +[Lambda-owned listing](#ownership-per-tree). `--prs` and git-ref download that listing, then +match by **filename-derived PR numbers or YAML `prs:`** (the same join git-ref uses). `--files` +/ a path list GETs those basenames only and does not read the registry. CDN `--all` and +product-only filters (no PR, issue, or file identity) are not supported yet — pass +`--force-local`. Local `--all` is unchanged. Under the artifact-root layout, entries are org/repo/branch-scoped — not product-scoped — so CDN entry sourcing keys off the resolvable authoring pool (repo with the same precedence as upload: diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs index c5eb895480..cc71c47251 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/CdnChangelogEntryFetcher.cs @@ -16,9 +16,11 @@ namespace Elastic.Documentation.Configuration.ReleaseNotes; /// /// Fetches the individual (scrubbed) changelog entries for a single authoring org/repo/branch pool from /// the public CDN, for the changelog bundle command when sourcing entries from S3 rather than a -/// local folder. It reads {base}/changelog/{org}/{repo}/{branch}/registry.json to enumerate entries -/// and downloads each {base}/changelog/{org}/{repo}/{branch}/{file} as raw YAML; the bundle command -/// then applies its usual filter (products / prs / issues) to the downloaded set. +/// local folder. enumerates via the pool registry.json then downloads +/// each listed YAML. GETs requested basenames only (used by +/// --files / a path list) and does not read the registry. The bundle command then applies its +/// usual filter (products / prs / issues) to the downloaded set, except --files which includes +/// every fetched name. /// /// /// @@ -39,6 +41,7 @@ public sealed class CdnChangelogEntryFetcher : IDisposable private const int DefaultMaxAttempts = 4; private const int BaseRetryDelayMs = 500; private const int MaxRetryDelayMs = 2000; + private const int MaxParallelNamedReads = 4; /// /// Bounds an individual registry/entry HTTP request so a stalled CDN connection cannot hang a bundle run. @@ -178,6 +181,79 @@ public async Task> FetchAsync( return entries; } + /// + /// Downloads only the named changelog entries from the authoring pool, without reading + /// registry.json. Used by changelog bundle --files / a path list so a stale or + /// missing pool listing cannot hide a requested object. A requested name that 404s after the + /// retry budget is a hard error (fail-fast, same as a missing local file). + /// + public async Task> FetchNamedAsync( + Uri baseUri, + string org, + string repo, + string branch, + IReadOnlyList fileNames, + Action emitError, + Action emitWarning, + Cancel ctx) + { + _ = emitWarning; + var poolLabel = $"{org}/{repo}/{branch}"; + + if (!ChangelogKeys.IsValidOrg(org) || !ChangelogKeys.IsValidRepo(repo) || !ChangelogKeys.IsValidBranch(branch)) + { + emitError( + $"Invalid changelog pool '{poolLabel}': the org, repo, and each '/'-delimited branch segment must be non-empty ASCII letters, digits, '.', '_' or '-' (org allows only letters, digits and '-') and must not be '.' or '..'."); + return []; + } + + var poolSegments = ChangelogKeys.PoolSegments(org, repo, branch); + var built = new CdnChangelogEntry?[fileNames.Count]; + var errors = new string?[fileNames.Count]; + + await Parallel.ForEachAsync( + Enumerable.Range(0, fileNames.Count), + new ParallelOptions { MaxDegreeOfParallelism = MaxParallelNamedReads, CancellationToken = ctx }, + async (i, ct) => + { + var fileName = fileNames[i]; + if (!ChangelogKeys.IsSafeFileName(fileName)) + { + errors[i] = + $"Changelog entry '{fileName}' for '{poolLabel}' is not a valid pool file name."; + return; + } + + var entryUri = CombineSegments(baseUri, [.. poolSegments, fileName]); + var (fetched, content, lastError) = await TryFetchEntryAsync(entryUri, fileName, poolLabel, ct).ConfigureAwait(false); + if (fetched) + { + built[i] = new CdnChangelogEntry(fileName, content); + return; + } + + errors[i] = + $"Changelog entry '{fileName}' for '{poolLabel}' could not be fetched from {entryUri} after {_maxAttempts} attempt(s): {lastError}. " + + "Ensure the entry was uploaded (changelog upload), or pass --force-local / --directory to bundle local files instead."; + }).ConfigureAwait(false); + + var failed = false; + for (var i = 0; i < errors.Length; i++) + { + if (errors[i] is not { } message) + continue; + emitError(message); + failed = true; + } + + if (failed) + return []; + + var entries = built.Where(e => e is not null).Select(e => e!.Value).ToList(); + _logger.LogInformation("Fetched {Count} named changelog entry(ies) for {Pool} from {BaseUri}", entries.Count, poolLabel, baseUri); + return entries; + } + /// /// Fetches a single entry, retrying transient failures (most importantly a not-yet-propagated 404) /// up to times with exponential backoff. Retry requests are cache-busted diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs index dbbe3e22d9..e6dac1f5c8 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogKeys.cs @@ -12,7 +12,7 @@ namespace Elastic.Documentation.Configuration.ReleaseNotes; /// changelog/{org}/{repo}/{branch}/{file}, and each grouping has a registry.json /// manifest at its root. Centralizes key construction, group extraction, and per-segment /// validation so the producer (ChangelogUploadService), the scrubber Lambda gate, -/// the registry builder, and the CDN fetchers cannot drift apart. +/// the registry reconciler, and the CDN fetchers cannot drift apart. /// /// /// Segment character classes, from strictest to loosest: org is a GitHub login diff --git a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs index d7f15e0189..046e91da59 100644 --- a/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs +++ b/src/Elastic.Documentation.Configuration/ReleaseNotes/ChangelogRegistry.cs @@ -9,7 +9,7 @@ namespace Elastic.Documentation.Configuration.ReleaseNotes; /// /// Consumer-side view of the bundle/{product}/registry.json manifest (or the /// changelog/{org}/{repo}/{branch}/registry.json entry index) published alongside scrubbed changelog content. -/// Mirrors the producer's shape (see the changelog upload service) but is intentionally lenient: only +/// Mirrors the producer's shape (the scrubber Lambda's registry reconciler) but is intentionally lenient: only /// the fields the changelog directive needs to enumerate bundles are declared, and nothing is /// required so a partially-written or future-versioned manifest still deserializes. /// diff --git a/src/infra/docs-lambda-changelog-scrubber/README.md b/src/infra/docs-lambda-changelog-scrubber/README.md index 9326eafe88..8d26603965 100644 --- a/src/infra/docs-lambda-changelog-scrubber/README.md +++ b/src/infra/docs-lambda-changelog-scrubber/README.md @@ -3,8 +3,8 @@ SQS-triggered Lambda that reads changelog/bundle YAML from the private S3 bucket, scrubs private repository references using `LinkAllowlistSanitizer`, writes sanitized copies to the public S3 bucket, and is the **sole producer** of the public -`bundle/{product}/registry.json` manifests and the shallow per-tree change maps, -reconciled from actual public bucket state +`bundle/{product}/registry.json` and `changelog/{org}/{repo}/{branch}/registry.json` +manifests and the shallow per-tree change maps, reconciled from actual public bucket state ([elastic/docs-eng-team#688](https://github.com/elastic/docs-eng-team/issues/688)). The handler logic lives in `Elastic.Changelog` (`Scrubbing/ScrubberProcessor`, `Reconciliation/BundleRegistryReconciler`, `Reconciliation/ShallowRegistryReconciler`); @@ -46,19 +46,18 @@ The `bootstrap` binary should be available under: S3 events are *triggers, not instructions* — the event type is ignored and current bucket state decides (events are at-least-once and can arrive out of order). Work is coalesced per SQS batch: one object reconcile per distinct key, one registry reconcile per distinct -`bundle/{product}/` group, one shallow-map reconcile per touched tree. +group (`bundle/{product}/` or `changelog/{org}/{repo}/{branch}/`), one shallow-map reconcile +per touched tree. - **`.yaml`/`.yml` keys**: object-level reconcile — GET the key from the private bucket; present → scrub the current content and PUT to public, absent → conditionally delete the public copy. A post-write HEAD re-validates the source and redoes the work if a concurrent - invocation raced it. A `bundle/{product}/` key then gets its group's `registry.json` - reconciled from the public listing, and the touched tree's shallow folder→token map is - patched. -- **Registry keys** (`ChangelogKeys.IsRegistry`): the trees part ways. A bundle manifest is - never copied or deleted — the event only schedules the group reconcile, so client-authored - JSON never reaches the tree consumers enumerate. A changelog pool manifest is mirrored - verbatim (legacy pass-through: `changelog bundle` still enumerates a pool through its - manifest until PR-list discovery replaces it). + invocation raced it. The key's group then gets its `registry.json` reconciled from the + public listing, and the touched tree's shallow folder→token map is patched. +- **Registry keys** (`ChangelogKeys.IsRegistry`): never copied or deleted — the event only + schedules the group reconcile, so client-authored JSON never reaches the tree consumers + enumerate. Pool listings are listing-only (`target` is null); bundle listings record + each file's `target`. - **Other `.json` keys**: skipped with a warning; other extensions are skipped silently. Registry and shallow-map writes use conditional PUT/DELETE (`If-Match` / `If-None-Match: *`) diff --git a/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs b/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs index c9b9e65bda..e0d23bbab5 100644 --- a/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs +++ b/src/services/Elastic.Changelog/Bundling/ChangelogBundlingService.cs @@ -346,7 +346,7 @@ public async Task BundleChangelogs(IDiagnosticsCollector collector, Bundle // Source and match changelog entries — from the CDN (default) or the local folder. // Explicit --files / path-list selection bypasses content filters (IncludeAll): locally it loads - // the named paths, in CDN mode it selects pool entries by file name. + // the named paths, in CDN mode it GETs those pool objects by file name (no registry listing). var entryMatcher = new ChangelogEntryMatcher(_fileSystem, ReleaseNotesSerialization.GetEntryDeserializer(), _logger); ChangelogMatchResult matchResult; if (explicitFilePaths != null) @@ -357,21 +357,32 @@ public async Task BundleChangelogs(IDiagnosticsCollector collector, Bundle } else if (useCdn) { - var contents = await FetchCdnEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, ctx); - if (contents == null) + if (requestedEntryNames is null && prsToMatch.Count == 0 && issuesToMatch.Count == 0) + { + collector.EmitError(string.Empty, + "CDN entry sourcing requires a PR, issue, or file identity (--prs, --issues, --files, a URL/path list, or --start-git-ref). " + + "--all and product-only filters are not supported on the CDN yet; pass --force-local to read the local folder."); return false; + } + + IReadOnlyList<(string FileName, string Content)>? contents; if (requestedEntryNames is not null) { - var poolLabel = $"{authoringOwner}/{authoringRepo}/{authoringBranch}"; - var selected = SelectRequestedCdnEntries(collector, contents, requestedEntryNames, poolLabel); - if (selected == null) + contents = await FetchCdnNamedEntriesAsync( + collector, authoringOwner, authoringRepo, authoringBranch, requestedEntryNames, ctx); + if (contents == null) return false; - _logger.LogInformation("Matching {Count} explicitly selected changelog entries from the CDN", selected.Count); + _logger.LogInformation("Matching {Count} explicitly selected changelog entries from the CDN", contents.Count); var filesCriteria = filterCriteria with { IncludeAll = true }; - matchResult = entryMatcher.MatchChangelogContents(collector, selected, filesCriteria, ctx); + matchResult = entryMatcher.MatchChangelogContents(collector, contents, filesCriteria, ctx); } else + { + contents = await FetchCdnEntriesAsync(collector, authoringOwner, authoringRepo, authoringBranch, ctx); + if (contents == null) + return false; matchResult = entryMatcher.MatchChangelogContents(collector, contents, filterCriteria, ctx); + } } else { @@ -1152,55 +1163,64 @@ private BundleChangelogsArguments ApplyConfigDefaults(BundleChangelogsArguments return byName.Select(kv => (kv.Key, kv.Value)).ToList(); } - /// Gate for repo-scoped CDN entry sourcing: true when the authoring repo resolves, local sourcing is not forced (bundle.use_local_changelogs/--force-local/--directory), and a CDN base is configured. - private static bool ShouldSourceFromCdn(string? authoringRepo, bool useLocalChangelogs, bool explicitDirectory) - { - if (useLocalChangelogs || explicitDirectory || string.IsNullOrWhiteSpace(authoringRepo)) - return false; - return ChangelogCdn.ResolveBaseUri() is not null; - } - /// - /// Selects the CDN-sourced entries whose file names were explicitly requested via --files / a - /// path list. Every requested name must exist in the pool: the registry is the source of truth for - /// what was uploaded, so a missing name means the entry never reached S3 (or the name is wrong) and - /// silently shipping an incomplete bundle is worse than failing the run. Returns null after - /// emitting an error when any requested name is missing. + /// Downloads only the requested entry names from the CDN pool (no registry listing). Returns + /// null after emitting an error when any requested name cannot be fetched. /// - private IReadOnlyList<(string FileName, string Content)>? SelectRequestedCdnEntries( + private async Task?> FetchCdnNamedEntriesAsync( IDiagnosticsCollector collector, - IReadOnlyList<(string FileName, string Content)> contents, - IReadOnlyList requestedEntryNames, - string poolLabel) + string? org, + string? repo, + string? branch, + IReadOnlyList fileNames, + Cancel ctx) { - var byName = new Dictionary(StringComparer.Ordinal); - foreach (var (fileName, content) in contents) - byName[fileName] = content; - - var selected = new List<(string FileName, string Content)>(); - var missing = new List(); - var seen = new HashSet(StringComparer.Ordinal); - foreach (var name in requestedEntryNames) + if (string.IsNullOrWhiteSpace(repo)) { - if (!seen.Add(name)) - continue; - if (byName.TryGetValue(name, out var content)) - selected.Add((name, content)); - else - missing.Add(name); + collector.EmitError(string.Empty, + "Sourcing changelog entries from the CDN requires a resolvable authoring repository. " + + "Set bundle.repo in changelog.yml (or pass --repo), or set bundle.use_local_changelogs: true " + + "in changelog.yml / pass --directory to bundle local changelog files."); + return null; } - if (missing.Count > 0) + var resolvedOrg = string.IsNullOrWhiteSpace(org) ? DefaultOwner : org; + var resolvedBranch = string.IsNullOrWhiteSpace(branch) ? DefaultBranch : branch; + + var baseUri = ChangelogCdn.ResolveBaseUri(); + if (baseUri is null) { collector.EmitError(string.Empty, - $"Changelog entr{(missing.Count == 1 ? "y" : "ies")} not found in the CDN pool '{poolLabel}': {string.Join(", ", missing)}. " + - "Ensure the entries were uploaded (changelog upload), or pass --force-local / --directory to bundle local files instead."); + $"No valid changelog CDN base URL is configured. Set the {ChangelogCdn.BaseUrlEnvironmentVariable} environment variable to an absolute http(s) URL."); return null; } - _logger.LogInformation("Selected {Selected} of {Total} CDN entries by requested file name for {Pool}", - selected.Count, contents.Count, poolLabel); - return selected; + var fatalFailure = false; + var entries = await _entryFetcher.FetchNamedAsync( + baseUri, + resolvedOrg, + repo, + resolvedBranch, + fileNames, + msg => { fatalFailure = true; collector.EmitError(string.Empty, msg); }, + msg => collector.EmitWarning(string.Empty, msg), + ctx); + + if (fatalFailure) + return null; + + _logger.LogInformation("Sourced {Count} named changelog entr(ies) from the CDN for {Pool}", + entries.Count, $"{resolvedOrg}/{repo}/{resolvedBranch}"); + + return entries.Select(e => (e.FileName, e.Content)).ToList(); + } + + /// Gate for repo-scoped CDN entry sourcing: true when the authoring repo resolves, local sourcing is not forced (bundle.use_local_changelogs/--force-local/--directory), and a CDN base is configured. + private static bool ShouldSourceFromCdn(string? authoringRepo, bool useLocalChangelogs, bool explicitDirectory) + { + if (useLocalChangelogs || explicitDirectory || string.IsNullOrWhiteSpace(authoringRepo)) + return false; + return ChangelogCdn.ResolveBaseUri() is not null; } private bool ValidateInput(IDiagnosticsCollector collector, BundleChangelogsArguments input, bool requireDirectoryExists) diff --git a/src/services/Elastic.Changelog/Bundling/ChangelogEntryMatcher.cs b/src/services/Elastic.Changelog/Bundling/ChangelogEntryMatcher.cs index 56f9f259ab..741fd63557 100644 --- a/src/services/Elastic.Changelog/Bundling/ChangelogEntryMatcher.cs +++ b/src/services/Elastic.Changelog/Bundling/ChangelogEntryMatcher.cs @@ -147,7 +147,7 @@ private static ChangelogMatchResult BuildResult( return null; } - if (!MatchesFilter(yamlDto, criteria, matchedPrs, matchedIssues)) + if (!MatchesFilter(yamlDto, fileName, criteria, matchedPrs, matchedIssues)) return null; // Add to seen set @@ -180,6 +180,7 @@ private static ChangelogMatchResult BuildResult( private static bool MatchesFilter( ChangelogEntryDto data, + string fileName, ChangelogFilterCriteria criteria, HashSet matchedPrs, HashSet matchedIssues) @@ -191,7 +192,7 @@ private static bool MatchesFilter( return MatchesProductFilter(data, criteria.ProductFilters); if (criteria.PrsToMatch.Count > 0) - return MatchesPrFilter(data, criteria, matchedPrs); + return MatchesPrFilter(data, fileName, criteria, matchedPrs); if (criteria.IssuesToMatch.Count > 0) return MatchesIssueFilter(data, criteria, matchedIssues); @@ -225,9 +226,22 @@ private static bool MatchesProductFilter( private static bool MatchesPrFilter( ChangelogEntryDto data, + string fileName, ChangelogFilterCriteria criteria, HashSet matchedPrs) { + var fileNumbers = ChangelogPrIdentity.ParseLeadingPrNumbers(fileName); + foreach (var pr in criteria.PrsToMatch) + { + var normalizedPrToMatch = ChangelogBundlingService.NormalizePrForComparison(pr, criteria.DefaultOwner, criteria.DefaultRepo); + if (ChangelogPrIdentity.TryParseNumberFromNormalized(normalizedPrToMatch, out var prNumber) + && fileNumbers.Contains(prNumber)) + { + _ = matchedPrs.Add(pr); + return true; + } + } + var prs = data.Prs ?? (data.Pr != null ? [data.Pr] : null); if (prs is not { Count: > 0 }) return false; diff --git a/src/services/Elastic.Changelog/Bundling/ChangelogPrIdentity.cs b/src/services/Elastic.Changelog/Bundling/ChangelogPrIdentity.cs new file mode 100644 index 0000000000..67265ce6c9 --- /dev/null +++ b/src/services/Elastic.Changelog/Bundling/ChangelogPrIdentity.cs @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Changelog.Bundling; + +/// +/// Shared identity for matching a changelog entry to a pull request: leading numeric filename +/// segments (survive scrubbing) or normalized YAML prs: references. +/// +internal static class ChangelogPrIdentity +{ + /// + /// Parses PR numbers from the leading dash-separated numeric segments of an entry file name, + /// covering the PR-number naming schemes (123.yaml, 123-456.yaml, + /// 123-bug-fix-slug.yaml). File names survive scrubbing, so this match works for + /// private pools whose prs references were removed from the public copies. + /// + public static IReadOnlyList ParseLeadingPrNumbers(string fileName) + { + var stem = fileName; + var extensionIndex = stem.LastIndexOf('.'); + if (extensionIndex > 0) + stem = stem[..extensionIndex]; + + var numbers = new List(); + foreach (var segment in stem.Split('-')) + { + if (segment.Length > 0 && segment.All(char.IsAsciiDigit) && int.TryParse(segment, out var number)) + numbers.Add(number); + else + break; + } + + return numbers; + } + + /// + /// Extracts the PR number from a value already normalized by + /// (owner/repo#n). + /// + public static bool TryParseNumberFromNormalized(string normalized, out int number) + { + number = 0; + var hash = normalized.LastIndexOf('#'); + if (hash < 0 || hash == normalized.Length - 1) + return false; + return int.TryParse(normalized[(hash + 1)..], out number); + } +} diff --git a/src/services/Elastic.Changelog/Bundling/GitRangeEntryResolver.cs b/src/services/Elastic.Changelog/Bundling/GitRangeEntryResolver.cs index f4f4ed86fb..dbb3e05c83 100644 --- a/src/services/Elastic.Changelog/Bundling/GitRangeEntryResolver.cs +++ b/src/services/Elastic.Changelog/Bundling/GitRangeEntryResolver.cs @@ -218,7 +218,7 @@ private sealed record ParsedCandidate(string FileName, IReadOnlyList FileNa private static ParsedCandidate ParseCandidate(string fileName, string content) { - var numbers = ParseLeadingPrNumbers(fileName); + var numbers = ChangelogPrIdentity.ParseLeadingPrNumbers(fileName); try { var checksum = ChangelogBundlingService.ComputeSha1(content); @@ -240,30 +240,9 @@ private static ParsedCandidate ParseCandidate(string fileName, string content) } } - /// - /// Parses PR numbers from the leading dash-separated numeric segments of an entry file name, - /// covering the PR-number naming schemes (123.yaml, 123-456.yaml, - /// 123-bug-fix-slug.yaml). File names survive scrubbing, so this match works for - /// private pools whose prs references were removed from the public copies. - /// - internal static IReadOnlyList ParseLeadingPrNumbers(string fileName) - { - var stem = fileName; - var extensionIndex = stem.LastIndexOf('.'); - if (extensionIndex > 0) - stem = stem[..extensionIndex]; - - var numbers = new List(); - foreach (var segment in stem.Split('-')) - { - if (segment.Length > 0 && segment.All(char.IsAsciiDigit) && int.TryParse(segment, out var number)) - numbers.Add(number); - else - break; - } - - return numbers; - } + /// + internal static IReadOnlyList ParseLeadingPrNumbers(string fileName) => + ChangelogPrIdentity.ParseLeadingPrNumbers(fileName); private static bool MatchesPr(ParsedCandidate candidate, int prNumber, GitRangeEntryResolutionOptions options) { diff --git a/src/services/Elastic.Changelog/Reconciliation/BundleRegistryReconciler.cs b/src/services/Elastic.Changelog/Reconciliation/BundleRegistryReconciler.cs index 526854f54f..4d32b691dd 100644 --- a/src/services/Elastic.Changelog/Reconciliation/BundleRegistryReconciler.cs +++ b/src/services/Elastic.Changelog/Reconciliation/BundleRegistryReconciler.cs @@ -45,11 +45,10 @@ public sealed class ReconcileConflictException(string message) : Exception(messa /// the change that triggered it. /// /// -/// Scoped to the bundle/{product}/ tree only: the {changelog} directive and external -/// CDN consumers need to enumerate bundles, and dates (serverless) are not derivable client-side. -/// The changelog/… pool manifests are deliberately not reconciled — release-note -/// discovery starts from PR lists, so those manifests stay client-authored pass-through until -/// Phase 3 retires them entirely. +/// Applies to both trees. Bundle manifests record a target per file (the +/// {changelog} directive enumerates by version/date). Pool manifests are listing-only: +/// target is always null — changelog bundle --prs and git-ref still need a complete +/// candidate listing because the public CDN cannot ListObjects. /// public sealed class BundleRegistryReconciler( ILoggerFactory logFactory, @@ -85,9 +84,6 @@ public sealed class BundleRegistryReconciler( /// public async Task ReconcileGroupAsync(ChangelogScope scope, Cancel ctx) { - if (scope.Kind != ChangelogScopeKind.Bundle) - throw new ArgumentException($"Group reconcile applies to the bundle tree only; got '{scope}'.", nameof(scope)); - _metrics.IncrementGroupReconciles(); for (var attempt = 1; attempt <= MaxWriteAttempts; attempt++) @@ -253,9 +249,10 @@ await Parallel.ForEachAsync( var file = obj.Key[scope.Prefix.Length..]; var etag = NormalizeETag(obj.ETag); - // Amends are never ETag-skipped: their target depends on the parent bundle too, - // and a parent appearing or changing does not touch the amend's own ETag. - if (!BundleAmendMerger.IsAmendFile(file) + // Bundle amends are never ETag-skipped: their target depends on the parent bundle too, + // and a parent appearing or changing does not touch the amend's own ETag. Pool + // listings have no target, so every YAML is ETag-skippable. + if ((scope.Kind == ChangelogScopeKind.Changelog || !BundleAmendMerger.IsAmendFile(file)) && byFile.TryGetValue(file, out var previous) && string.Equals(previous.ETag, etag, StringComparison.Ordinal)) { @@ -264,7 +261,9 @@ await Parallel.ForEachAsync( return; } - var target = await ComputeTarget(scope, file, ct); + var target = scope.Kind == ChangelogScopeKind.Bundle + ? await ComputeTarget(scope, file, ct) + : null; _metrics.IncrementEntriesRecomputed(); built[i] = new RegistryBundle { File = file, Target = target, ETag = etag }; }); diff --git a/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs b/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs index bef62e55b7..fa7c9d120e 100644 --- a/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs +++ b/src/services/Elastic.Changelog/Scrubbing/ScrubberProcessor.cs @@ -20,9 +20,10 @@ public sealed record ScrubberQueueMessage(string MessageId, string Body); /// Program.cs so it is testable. Events are triggers, state decides: the handler never /// acts on an event's type — an event means only "this key may have changed, look at /// it". Every distinct key gets one object-level reconcile against the private bucket; every -/// distinct bundle/{product}/ group then gets one registry reconcile against the public -/// listing, and every touched tree gets one shallow-map reconcile. Out-of-order and -/// at-least-once S3 notifications are harmless and each batch heals accumulated drift. +/// distinct group then gets one registry reconcile against the public listing (bundle +/// target metadata; pool listing-only), and every touched tree gets one +/// shallow-map reconcile. Out-of-order and at-least-once S3 notifications are harmless +/// and each batch heals accumulated drift. /// public sealed class ScrubberProcessor( ILoggerFactory logFactory, @@ -46,7 +47,7 @@ private sealed class ObjectWork(string sourceBucket, bool passThrough) { public string SourceBucket { get; set; } = sourceBucket; - /// True for a pool manifest copied verbatim; false for YAML content that is scrubbed. + /// True to copy JSON verbatim; false to scrub YAML. Registry keys no longer use this path. public bool PassThrough { get; } = passThrough; public HashSet MessageIds { get; } = [with(StringComparer.Ordinal)]; @@ -158,15 +159,9 @@ private void Classify( if (!hasScope) return; - // The two trees part ways here. Bundle manifests are reconciler-owned: the event only - // schedules a group reconcile, so client-authored JSON never reaches the public bucket - // for the tree consumers enumerate. Pool manifests stay client-authored pass-through — - // `changelog bundle` still enumerates a pool through its manifest today, and 404-probing - // only works once entries are guaranteed one-per-PR — until Phase 3 retires them. - if (scope!.Kind == ChangelogScopeKind.Bundle) - AddGroup(groupWork, scope, messageId); - else - AddObject(objectWork, key, sourceBucket, messageId, passThrough: true); + // Both trees are reconciler-owned: the event only schedules a group reconcile, so + // client-authored JSON never reaches the public bucket consumers enumerate. + AddGroup(groupWork, scope!, messageId); return; } @@ -188,9 +183,8 @@ private void Classify( if (!hasScope) return; - if (scope!.Kind == ChangelogScopeKind.Bundle) - AddGroup(groupWork, scope, messageId); - AddShallow(shallowWork, scope, messageId); + AddGroup(groupWork, scope!, messageId); + AddShallow(shallowWork, scope!, messageId); } private static void AddObject( @@ -234,7 +228,7 @@ private static void AddShallow(Dictionary shall /// Order-independent object reconcile: the event type is ignored; the private bucket's current /// state decides between copy and delete. A stale ObjectRemoved arriving after a /// recreate re-copies the live object instead of deleting it. YAML content is scrubbed on the - /// way through; a pass-through pool manifest is copied verbatim. + /// way through. Pass-through remains for unused JSON copy of non-registry keys. /// private async Task ReconcileObjectAsync(string sourceBucket, string key, bool passThrough, Cancel ctx) { diff --git a/src/services/Elastic.Changelog/Uploading/Registry.cs b/src/services/Elastic.Changelog/Uploading/Registry.cs index 17d16f3ee3..6970e2398c 100644 --- a/src/services/Elastic.Changelog/Uploading/Registry.cs +++ b/src/services/Elastic.Changelog/Uploading/Registry.cs @@ -14,10 +14,9 @@ namespace Elastic.Changelog.Uploading; /// /// Stored at bundle/{product}/registry.json (bundle index) or /// changelog/{org}/{repo}/{branch}/registry.json (changelog-entry index). Ownership differs -/// per tree: public bundle indexes are produced by the scrubber Lambda's -/// from public-bucket state, while -/// changelog-entry indexes remain client-authored and are mirrored verbatim to the public bucket -/// (pass-through) until Phase 3 of elastic/docs-eng-team#688 retires them. +/// per tree: public indexes on both trees are produced by the scrubber Lambda's +/// from public-bucket state. Upload writes +/// YAML only; it never writes a registry.json. /// public sealed record Registry { @@ -80,14 +79,12 @@ public sealed record RegistryBundle /// the MD5 of the body. /// /// - /// Whose ETag this is follows the manifest's producer. Bundle indexes written by the scrubber + /// Whose ETag this is follows the manifest's producer. Indexes written by the scrubber /// Lambda's record the public - /// (post-scrub) object's ETag, valid for HTTP cache validation against the CDN. Client-authored - /// manifests — changelog-entry indexes, and everything in the private bucket — record the - /// private (pre-scrub) upload's ETag: a best-effort change hint that will not - /// match the public object for scrubbed content, so consumers MUST NOT use it for integrity - /// checks or cache validation there. Either way it is safe for detecting that an entry changed - /// between manifest reads. + /// (post-scrub) object's ETag, valid for HTTP cache validation against the CDN. Legacy + /// client-authored manifests in the private bucket (no longer produced) recorded the + /// private (pre-scrub) upload's ETag. Either way it is safe for detecting that an + /// entry changed between manifest reads. /// public required string ETag { get; init; } } diff --git a/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs index 63b770eae0..3cd083e03f 100644 --- a/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs +++ b/tests/Elastic.Changelog.Tests/Changelogs/BundleCdnSourcingTests.cs @@ -68,18 +68,25 @@ private static CdnChangelogEntryFetcher Fetcher(ITestOutputHelper output, StubHa private string OutputPath() => FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString(), "bundle.yaml"); + private static string[] BothPrs => + [ + "https://github.com/elastic/elasticsearch/pull/100", + "https://github.com/elastic/elasticsearch/pull/999" + ]; + [Fact] - public async Task OptionMode_RepoResolvable_SourcesAllEntriesFromRepoPoolOnCdn() + public async Task OptionMode_RepoResolvable_SourcesMatchingPrsFromRepoPoolOnCdn() { // Under the artifact-root layout the CDN entry pool is keyed by the authoring repo, not the - // target product. A resolvable repo (here via --repo) is what enables CDN sourcing. + // target product. A resolvable repo (here via --repo) is what enables CDN sourcing. --prs + // downloads the pool via registry.json then matches filename digits or YAML prs:. var handler = RegistryHandler(); var service = new ChangelogBundlingService(LoggerFactory, FileSystem, null, null, Fetcher(Output, handler)); var output = OutputPath(); var input = new BundleChangelogsArguments { - InputProducts = [new ProductArgument { Product = "elasticsearch", Target = "*", Lifecycle = "*" }], + Prs = BothPrs, Output = output, Repo = "elasticsearch" }; @@ -89,7 +96,6 @@ public async Task OptionMode_RepoResolvable_SourcesAllEntriesFromRepoPoolOnCdn() result.Should().BeTrue($"Errors: {string.Join("; ", Collector.Diagnostics.Where(d => d.Severity == Severity.Error).Select(d => d.Message))}"); Collector.Errors.Should().Be(0); - // Entries are sourced from the authoring pool, with org/branch defaulting: changelog/{org}/{repo}/{branch}/... handler.RequestedPaths.Should().Contain("/changelog/elastic/elasticsearch/main/registry.json"); var bundle = await FileSystem.File.ReadAllTextAsync(output, TestContext.Current.CancellationToken); @@ -108,7 +114,7 @@ public async Task OptionMode_OwnerAndBranchOverride_SourcesFromThatPoolOnCdn() var input = new BundleChangelogsArguments { - InputProducts = [new ProductArgument { Product = "elasticsearch", Target = "*", Lifecycle = "*" }], + Prs = BothPrs, Output = output, Owner = "acme-corp", Repo = "elasticsearch", @@ -133,7 +139,7 @@ public async Task OptionMode_OwnerFromCombinedRepo_SourcesFromThatPool() var input = new BundleChangelogsArguments { - InputProducts = [new ProductArgument { Product = "elasticsearch", Target = "*", Lifecycle = "*" }], + Prs = BothPrs, Output = output, Repo = "acme-corp/widget" }; @@ -230,7 +236,7 @@ public async Task RegistryFailure_FailsBundle() var input = new BundleChangelogsArguments { - InputProducts = [new ProductArgument { Product = "elasticsearch", Target = "*", Lifecycle = "*" }], + Prs = BothPrs, Output = OutputPath(), Repo = "elasticsearch" }; @@ -260,7 +266,7 @@ public async Task EntryMissingAfterRetries_FailsBundle() var input = new BundleChangelogsArguments { - InputProducts = [new ProductArgument { Product = "elasticsearch", Target = "*", Lifecycle = "*" }], + Prs = BothPrs, Output = OutputPath(), Repo = "elasticsearch" }; @@ -322,6 +328,127 @@ public async Task ProfileGitHubRelease_ScopesByOutputProductsAndFiltersByRelease bundle.Should().NotContain("Bravo"); } + [Fact] + public async Task OptionMode_All_OnCdn_ErrorsAndDoesNotFetch() + { + var handler = RegistryHandler(); + var service = new ChangelogBundlingService(LoggerFactory, FileSystem, null, null, Fetcher(Output, handler)); + + var input = new BundleChangelogsArguments + { + All = true, + Output = OutputPath(), + Repo = "elasticsearch" + }; + + var result = await service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + Collector.Diagnostics.Should().Contain(d => + d.Severity == Severity.Error && d.Message.Contains("--force-local")); + handler.RequestedPaths.Should().BeEmpty("CDN --all must fail before any fetch"); + } + + [Fact] + public async Task OptionMode_InputProductsOnly_OnCdn_ErrorsAndDoesNotFetch() + { + var handler = RegistryHandler(); + var service = new ChangelogBundlingService(LoggerFactory, FileSystem, null, null, Fetcher(Output, handler)); + + var input = new BundleChangelogsArguments + { + InputProducts = [new ProductArgument { Product = "elasticsearch", Target = "*", Lifecycle = "*" }], + Output = OutputPath(), + Repo = "elasticsearch" + }; + + var result = await service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); + + result.Should().BeFalse(); + Collector.Diagnostics.Should().Contain(d => + d.Severity == Severity.Error && d.Message.Contains("--force-local")); + handler.RequestedPaths.Should().BeEmpty("CDN product-only filters must fail before any fetch"); + } + + [Fact] + public async Task OptionMode_Prs_MatchesFilenameDigitsWhenYamlPrsEmpty() + { + const string scrubbed = """ + title: Scrubbed + type: feature + products: + - product: elasticsearch + target: 9.3.0 + lifecycle: ga + """; + var handler = new StubHandler(req => + { + var path = req.RequestUri!.AbsolutePath; + if (path.EndsWith("/registry.json", StringComparison.Ordinal)) + return Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "12345.yaml" }, { "file": "1735-foo.yaml" } ] }"""); + if (path.EndsWith("12345.yaml", StringComparison.Ordinal)) + return Yaml(scrubbed); + if (path.EndsWith("1735-foo.yaml", StringComparison.Ordinal)) + return Yaml(scrubbed + "\nprs:\n - https://github.com/elastic/elasticsearch/pull/99999\n"); + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + var service = new ChangelogBundlingService(LoggerFactory, FileSystem, null, null, Fetcher(Output, handler)); + var output = OutputPath(); + + var input = new BundleChangelogsArguments + { + Prs = ["https://github.com/elastic/elasticsearch/pull/12345"], + Output = output, + Repo = "elasticsearch" + }; + + var result = await service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); + + result.Should().BeTrue($"Errors: {string.Join("; ", Collector.Diagnostics.Where(d => d.Severity == Severity.Error).Select(d => d.Message))}"); + var bundle = await FileSystem.File.ReadAllTextAsync(output, TestContext.Current.CancellationToken); + bundle.Should().Contain("name: 12345.yaml"); + bundle.Should().NotContain("name: 1735-foo.yaml"); + } + + [Fact] + public async Task OptionMode_Prs_MatchesTimestampFileViaYamlPrs() + { + const string timestamped = """ + title: Timestamped + type: feature + products: + - product: elasticsearch + target: 9.3.0 + lifecycle: ga + prs: + - https://github.com/elastic/elasticsearch/pull/12345 + """; + var handler = new StubHandler(req => + { + var path = req.RequestUri!.AbsolutePath; + if (path.EndsWith("/registry.json", StringComparison.Ordinal)) + return Json(/*lang=json,strict*/ """{ "schema_version": 1, "product": "elasticsearch", "bundles": [ { "file": "1735-foo.yaml" } ] }"""); + if (path.EndsWith("1735-foo.yaml", StringComparison.Ordinal)) + return Yaml(timestamped); + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + var service = new ChangelogBundlingService(LoggerFactory, FileSystem, null, null, Fetcher(Output, handler)); + var output = OutputPath(); + + var input = new BundleChangelogsArguments + { + Prs = ["https://github.com/elastic/elasticsearch/pull/12345"], + Output = output, + Repo = "elasticsearch" + }; + + var result = await service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); + + result.Should().BeTrue($"Errors: {string.Join("; ", Collector.Diagnostics.Where(d => d.Severity == Severity.Error).Select(d => d.Message))}"); + var bundle = await FileSystem.File.ReadAllTextAsync(output, TestContext.Current.CancellationToken); + bundle.Should().Contain("name: 1735-foo.yaml"); + } + private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK) { Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") }; diff --git a/tests/Elastic.Changelog.Tests/Changelogs/BundleChangelogsTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/BundleChangelogsTests.cs index 8f195f8c52..bae4c3d9a6 100644 --- a/tests/Elastic.Changelog.Tests/Changelogs/BundleChangelogsTests.cs +++ b/tests/Elastic.Changelog.Tests/Changelogs/BundleChangelogsTests.cs @@ -358,6 +358,68 @@ public async Task BundleChangelogs_WithPrsFilterAndUnmatchedPrs_EmitsWarnings() d.Message.Contains("No changelog file found for PR: https://github.com/elastic/elasticsearch/pull/300")); } + [Fact] + public async Task BundleChangelogs_WithPrsFilter_MatchesFilenameDigitsWhenYamlPrsEmpty() + { + var changelog = + """ + title: Filename identity + type: feature + products: + - product: elasticsearch + target: 9.2.0 + """; + + var file = FileSystem.Path.Join(_changelogDir, "12345.yaml"); + await FileSystem.File.WriteAllTextAsync(file, changelog, TestContext.Current.CancellationToken); + + var input = new BundleChangelogsArguments + { + Directory = _changelogDir, + Prs = ["https://github.com/elastic/elasticsearch/pull/12345"], + Output = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString(), "bundle.yaml") + }; + + var result = await Service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + var bundleContent = await FileSystem.File.ReadAllTextAsync(input.Output, TestContext.Current.CancellationToken); + bundleContent.Should().Contain("name: 12345.yaml"); + } + + [Fact] + public async Task BundleChangelogs_WithPrsFilter_MatchesTimestampFileViaYamlPrs() + { + var changelog = + """ + title: Timestamp identity + type: feature + products: + - product: elasticsearch + target: 9.2.0 + prs: + - https://github.com/elastic/elasticsearch/pull/12345 + """; + + var file = FileSystem.Path.Join(_changelogDir, "1735-foo.yaml"); + await FileSystem.File.WriteAllTextAsync(file, changelog, TestContext.Current.CancellationToken); + + var input = new BundleChangelogsArguments + { + Directory = _changelogDir, + Prs = ["https://github.com/elastic/elasticsearch/pull/12345"], + Output = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString(), "bundle.yaml") + }; + + var result = await Service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); + + result.Should().BeTrue(); + Collector.Errors.Should().Be(0); + var bundleContent = await FileSystem.File.ReadAllTextAsync(input.Output, TestContext.Current.CancellationToken); + bundleContent.Should().Contain("name: 1735-foo.yaml"); + } + [Fact] public async Task BundleChangelogs_WithPrsFileFilter_FiltersCorrectly() { diff --git a/tests/Elastic.Changelog.Tests/Changelogs/BundleFilesFilterTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/BundleFilesFilterTests.cs index 7ba1ab2e2e..8ed6e5bfee 100644 --- a/tests/Elastic.Changelog.Tests/Changelogs/BundleFilesFilterTests.cs +++ b/tests/Elastic.Changelog.Tests/Changelogs/BundleFilesFilterTests.cs @@ -321,7 +321,9 @@ public async Task Bundle_WithFiles_RepoResolves_MatchesCdnPoolByFileName() var result = await service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); result.Should().BeTrue($"Errors: {string.Join("; ", Collector.Diagnostics.Select(d => d.Message))}"); - handler.RequestedPaths.Should().Contain("/changelog/elastic/elasticsearch/main/registry.json"); + handler.RequestedPaths.Should().NotContain(p => p.EndsWith("/registry.json", StringComparison.Ordinal)); + handler.RequestedPaths.Should().Contain("/changelog/elastic/elasticsearch/main/keep.yaml"); + handler.RequestedPaths.Should().NotContain(p => p.EndsWith("/skip.yaml", StringComparison.Ordinal)); var bundle = await FileSystem.File.ReadAllTextAsync(output, TestContext.Current.CancellationToken); bundle.Should().Contain("name: keep.yaml"); bundle.Should().NotContain("name: skip.yaml"); @@ -344,7 +346,7 @@ public async Task Bundle_WithFiles_CdnPoolMissingRequestedName_FailsBundle() result.Should().BeFalse(); Collector.Diagnostics.Should().Contain(d => - d.Severity == Severity.Error && d.Message.Contains("not found in the CDN pool") && d.Message.Contains("never-uploaded.yaml")); + d.Severity == Severity.Error && d.Message.Contains("could not be fetched") && d.Message.Contains("never-uploaded.yaml")); } [Fact] @@ -384,7 +386,9 @@ public async Task Bundle_WithProfile_PathListFile_RepoResolves_SourcesFromCdn() var result = await service.BundleChangelogs(Collector, input, TestContext.Current.CancellationToken); result.Should().BeTrue($"Errors: {string.Join("; ", Collector.Diagnostics.Select(d => d.Message))}"); - handler.RequestedPaths.Should().Contain("/changelog/elastic/elasticsearch/main/registry.json"); + handler.RequestedPaths.Should().NotContain(p => p.EndsWith("/registry.json", StringComparison.Ordinal)); + handler.RequestedPaths.Should().Contain("/changelog/elastic/elasticsearch/main/keep.yaml"); + handler.RequestedPaths.Should().NotContain(p => p.EndsWith("/skip.yaml", StringComparison.Ordinal)); var bundle = await FileSystem.File.ReadAllTextAsync( FileSystem.Path.Join(outputDir, "bundle.yaml"), TestContext.Current.CancellationToken); bundle.Should().Contain("name: keep.yaml"); diff --git a/tests/Elastic.Changelog.Tests/Changelogs/CloudProfileFixtureTests.cs b/tests/Elastic.Changelog.Tests/Changelogs/CloudProfileFixtureTests.cs index 32294e20b7..23b7b6a2b0 100644 --- a/tests/Elastic.Changelog.Tests/Changelogs/CloudProfileFixtureTests.cs +++ b/tests/Elastic.Changelog.Tests/Changelogs/CloudProfileFixtureTests.cs @@ -97,9 +97,15 @@ public async Task MonthlyProfile_SourcesFromRepoPool_ProducesSoundBundle() var outputDir = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString()); FileSystem.Directory.CreateDirectory(outputDir); + var localDir = FileSystem.Path.Join(Paths.WorkingDirectoryRoot.FullName, Guid.NewGuid().ToString(), "changelog"); + FileSystem.Directory.CreateDirectory(localDir); + await FileSystem.File.WriteAllTextAsync(FileSystem.Path.Join(localDir, "1-feature.yaml"), FeatureEntry, TestContext.Current.CancellationToken); + await FileSystem.File.WriteAllTextAsync(FileSystem.Path.Join(localDir, "2-docs.yaml"), DocsEntry, TestContext.Current.CancellationToken); + await FileSystem.File.WriteAllTextAsync(FileSystem.Path.Join(localDir, "3-other.yaml"), OtherProductEntry, TestContext.Current.CancellationToken); + // language=yaml var configContent = - """ + $$""" products: available: - cloud-hosted @@ -114,9 +120,11 @@ public async Task MonthlyProfile_SourcesFromRepoPool_ProducesSoundBundle() bundle: exclude_types: "docs" bundle: + directory: {{localDir}} output_directory: PLACEHOLDER repo: widget owner: elastic + use_local_changelogs: true release_dates: false link_allow_repos: - elastic/elasticsearch @@ -147,10 +155,9 @@ public async Task MonthlyProfile_SourcesFromRepoPool_ProducesSoundBundle() result.Should().BeTrue($"Errors: {string.Join("; ", Collector.Diagnostics.Where(d => d.Severity == Severity.Error).Select(d => d.Message))}"); Collector.Errors.Should().Be(0); - // Entries are sourced once from the authoring pool (changelog/{org}/{repo}/{branch}/...), not from - // any product-scoped path. Owner comes from bundle.owner; branch defaults to "main". - handler.RequestedPaths.Should().Contain($"/changelog/elastic/{AuthoringRepo}/main/registry.json"); - handler.RequestedPaths.Should().NotContain(p => p.Contains("/cloud-hosted/changelog/", StringComparison.Ordinal)); + // Product-only CDN sourcing is deferred; this fixture forces local so it still covers + // exclude_types, link-allowlist scrubbing, and release_dates: false. + handler.RequestedPaths.Should().BeEmpty("use_local_changelogs must not reach the CDN"); var outputFiles = FileSystem.Directory.GetFiles(outputDir, "*.yaml"); outputFiles.Should().ContainSingle("the monthly profile writes a single bundle file"); diff --git a/tests/Elastic.Changelog.Tests/Reconciliation/BundleRegistryReconcilerTests.cs b/tests/Elastic.Changelog.Tests/Reconciliation/BundleRegistryReconcilerTests.cs index 70c9851f00..6c0692faac 100644 --- a/tests/Elastic.Changelog.Tests/Reconciliation/BundleRegistryReconcilerTests.cs +++ b/tests/Elastic.Changelog.Tests/Reconciliation/BundleRegistryReconcilerTests.cs @@ -268,17 +268,53 @@ public async Task ReconcileGroup_ListingPaginates() } [Fact] - public async Task ReconcileGroup_ChangelogScope_IsRejected() + public async Task ReconcileGroup_ChangelogScope_WritesListingOnlyManifest() { - // Pool manifests are not reconciled: they stay client-authored pass-through until Phase 3 - // retires them, so a changelog scope reaching the group reconciler is a programming error. - var scope = ChangelogScopeFor("elastic", "repo", "main"); - _ = _s3.Seed(PublicBucket, scope.Prefix + "entry-a.yaml", "a: 1"); + var scope = ChangelogScopeFor("elastic", "kibana", "main"); + _ = _s3.Seed(PublicBucket, scope.Prefix + "100.yaml", "title: a"); + _ = _s3.Seed(PublicBucket, scope.Prefix + "1735-foo.yaml", "title: b"); - var act = async () => await _reconciler.ReconcileGroupAsync(scope, Ctx); + var outcome = await _reconciler.ReconcileGroupAsync(scope, Ctx); - _ = await act.Should().ThrowAsync(); - _s3.Puts.Should().BeEmpty(); + outcome.Should().Be(GroupReconcileOutcome.Written); + var content = _s3.ContentOf(PublicBucket, scope.RegistryKey); + var manifest = JsonSerializer.Deserialize(content, RegistryJsonContext.Default.Registry)!; + manifest.Producer.Should().Be(BundleRegistryReconciler.Producer); + manifest.Product.Should().Be("elastic/kibana/main"); + manifest.Bundles.Select(b => b.File).Should().Equal("100.yaml", "1735-foo.yaml"); + manifest.Bundles.Should().OnlyContain(b => b.Target == null); + _s3.GetsFor(PublicBucket).Should().BeEquivalentTo([scope.RegistryKey], + "pool listings do not GET YAML to compute a target"); + } + + [Fact] + public async Task ReconcileGroup_ChangelogScope_HealsYamlMissingFromStaleClientManifest() + { + var scope = ChangelogScopeFor("elastic", "kibana", "main"); + _ = _s3.Seed(PublicBucket, scope.Prefix + "100.yaml", "title: a"); + _ = _s3.Seed(PublicBucket, scope.Prefix + "200.yaml", "title: b"); + SeedManifest(scope, producer: null, schemaVersion: Registry.CurrentSchemaVersion, + new RegistryBundle { File = "100.yaml", Target = null, ETag = "stale" }); + + var outcome = await _reconciler.ReconcileGroupAsync(scope, Ctx); + + outcome.Should().Be(GroupReconcileOutcome.Written); + var content = _s3.ContentOf(PublicBucket, scope.RegistryKey); + var manifest = JsonSerializer.Deserialize(content, RegistryJsonContext.Default.Registry)!; + manifest.Bundles.Select(b => b.File).Should().Equal("100.yaml", "200.yaml"); + manifest.Producer.Should().Be(BundleRegistryReconciler.Producer); + } + + [Fact] + public async Task ReconcileGroup_ChangelogScope_EmptyGroup_DeletesTheManifest() + { + var scope = ChangelogScopeFor("elastic", "kibana", "main"); + SeedManifest(scope, producer: BundleRegistryReconciler.Producer, schemaVersion: Registry.CurrentSchemaVersion); + + var outcome = await _reconciler.ReconcileGroupAsync(scope, Ctx); + + outcome.Should().Be(GroupReconcileOutcome.Deleted); + _s3.Exists(PublicBucket, scope.RegistryKey).Should().BeFalse(); } [Fact] diff --git a/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs b/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs index 7ae9d72f49..35d2488a1d 100644 --- a/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs +++ b/tests/Elastic.Changelog.Tests/Scrubbing/ScrubberProcessorTests.cs @@ -122,28 +122,29 @@ public async Task Process_BundleRegistryKeyEvents_NeverCopyOrDelete_OnlyTriggerA } [Fact] - public async Task Process_PoolRegistryKeyEvents_ArePassedThroughVerbatim() + public async Task Process_PoolRegistryKeyEvents_NeverCopyOrDelete_OnlyTriggerAGroupReconcile() { - // Pool manifests stay client-authored until Phase 3: `changelog bundle` still enumerates a - // pool through its manifest, so the private copy is mirrored verbatim — never scrubbed, - // never reconciled. + // Pool manifests are reconciler-owned, same as bundles: the event only schedules a listing + // reconcile. Client JSON in the private bucket must never reach the public key. const string poolRegistry = "changelog/elastic/kibana/main/registry.json"; - const string content = /*lang=json,strict*/ """{"schema_version":1,"bundles":[{"file":"100.yaml"}]}"""; - _ = _s3.Seed(PrivateBucket, poolRegistry, content); + _ = _s3.Seed(PrivateBucket, poolRegistry, /*lang=json,strict*/ """{"schema_version":1,"bundles":[{"file":"stale.yaml"}]}"""); + _ = _s3.Seed(PublicBucket, "changelog/elastic/kibana/main/100.yaml", "scrubbed: entry"); var failed = await _processor.ProcessAsync([Message("ObjectCreated:Put", poolRegistry)], Ctx); failed.Should().BeEmpty(); - _s3.ContentOf(PublicBucket, poolRegistry).Should().Be(content, "pass-through must not transform the manifest"); - _metrics.GroupReconciles.Should().Be(0, "pool manifests are not reconciled"); - _s3.Puts.Single(p => p.Key == poolRegistry).ContentType.Should().Be("application/json"); + var manifest = PublicManifest(poolRegistry); + manifest.Producer.Should().Be(BundleRegistryReconciler.Producer); + manifest.Product.Should().Be("elastic/kibana/main"); + manifest.Bundles.Select(b => b.File).Should().Equal("100.yaml"); + manifest.Bundles.Should().OnlyContain(b => b.Target == null); + _s3.GetsFor(PrivateBucket).Should().BeEmpty("the private pool registry content must never be read"); } [Fact] - public async Task Process_PoolRegistryKeyEvents_WithPrivateGone_DeleteThePublicCopy() + public async Task Process_PoolRegistryKeyEvents_EmptyPublicGroup_DeletesTheManifest() { - // State decides for pass-through keys too: Phase 3's private-manifest cleanup deletes will - // propagate and remove the public pool manifests with them. + // A registry-key event with no public YAML is an empty-group reconcile: absent ≠ empty. const string poolRegistry = "changelog/elastic/kibana/main/registry.json"; _ = _s3.Seed(PublicBucket, poolRegistry, "{}"); @@ -154,7 +155,7 @@ public async Task Process_PoolRegistryKeyEvents_WithPrivateGone_DeleteThePublicC } [Fact] - public async Task Process_PoolYamlEvents_ScrubAndUpdateTheShallowMap_ButWriteNoPoolManifest() + public async Task Process_PoolYamlEvents_ScrubAndReconcileThePoolManifest() { _ = _s3.Seed(PrivateBucket, "changelog/elastic/kibana/main/100.yaml", "entry"); @@ -162,9 +163,11 @@ public async Task Process_PoolYamlEvents_ScrubAndUpdateTheShallowMap_ButWriteNoP failed.Should().BeEmpty(); _s3.ContentOf(PublicBucket, "changelog/elastic/kibana/main/100.yaml").Should().Be("scrubbed: entry"); - _s3.Exists(PublicBucket, "changelog/elastic/kibana/main/registry.json") - .Should().BeFalse("the reconciler no longer produces pool manifests"); - _metrics.GroupReconciles.Should().Be(0); + var manifest = PublicManifest("changelog/elastic/kibana/main/registry.json"); + manifest.Producer.Should().Be(BundleRegistryReconciler.Producer); + manifest.Bundles.Select(b => b.File).Should().Equal("100.yaml"); + manifest.Bundles.Should().OnlyContain(b => b.Target == null); + _metrics.GroupReconciles.Should().Be(1); var map = ShallowMap("changelog/registry.json"); map.Should().ContainKey("elastic/kibana/main"); diff --git a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs index 7d2f182786..f9fd63b3f3 100644 --- a/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs +++ b/tests/Elastic.Documentation.Configuration.Tests/ReleaseNotes/CdnChangelogEntryFetcherTests.cs @@ -188,6 +188,42 @@ public async Task FetchAsync_UnsafeFileName_EmitsWarningAndSkips() warnings.Should().ContainSingle().Which.Should().Contain("escape.yaml"); } + [Fact] + public async Task FetchNamedAsync_HappyPath_GetsOnlyRequestedFilesWithoutRegistry() + { + var handler = new StubHandler(req => + req.RequestUri!.AbsolutePath.EndsWith("/keep.yaml", StringComparison.Ordinal) + ? Yaml(SampleEntry) + : new HttpResponseMessage(HttpStatusCode.NotFound)); + var (errors, warnings, emitError, emitWarning) = Diagnostics(); + + using var fetcher = CreateFetcher(handler); + var entries = await fetcher.FetchNamedAsync( + BaseUri, "elastic", "elasticsearch", "main", ["keep.yaml"], emitError, emitWarning, TestContext.Current.CancellationToken); + + errors.Should().BeEmpty(); + warnings.Should().BeEmpty(); + entries.Select(e => e.FileName).Should().BeEquivalentTo("keep.yaml"); + handler.RequestedPaths.Should().NotContain(p => p.EndsWith("/registry.json", StringComparison.Ordinal)); + handler.RequestedPaths.Should().Contain("/changelog/elastic/elasticsearch/main/keep.yaml"); + } + + [Fact] + public async Task FetchNamedAsync_MissingAfterRetries_EmitsError() + { + var handler = new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + var (errors, _, emitError, emitWarning) = Diagnostics(); + + using var fetcher = CreateFetcher(handler, maxAttempts: 3); + var entries = await fetcher.FetchNamedAsync( + BaseUri, "elastic", "elasticsearch", "main", ["never-uploaded.yaml"], emitError, emitWarning, TestContext.Current.CancellationToken); + + entries.Should().BeEmpty(); + errors.Should().ContainSingle().Which.Should().Contain("never-uploaded.yaml"); + handler.RequestedPaths.Count(p => p.EndsWith("/never-uploaded.yaml", StringComparison.Ordinal)) + .Should().Be(3); + } + private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK) { Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") };