diff --git a/src/Elastic.ApiExplorer/Export/OpenApiDocumentExporter.cs b/src/Elastic.ApiExplorer/Export/OpenApiDocumentExporter.cs index 07b8e68b59..f5e96b4ba6 100644 --- a/src/Elastic.ApiExplorer/Export/OpenApiDocumentExporter.cs +++ b/src/Elastic.ApiExplorer/Export/OpenApiDocumentExporter.cs @@ -2,36 +2,35 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.Globalization; using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using Elastic.ApiExplorer.Infrastructure; using Elastic.ApiExplorer.Model; -using Elastic.ApiExplorer.Operations; using Elastic.Documentation; using Elastic.Documentation.AppliesTo; using Elastic.Documentation.Configuration.Inference; using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; using Elastic.Documentation.Search; using Elastic.Documentation.Search.Contract; using Elastic.Documentation.Versions; using Microsoft.OpenApi; -using Microsoft.OpenApi.Reader; namespace Elastic.ApiExplorer.Export; /// -/// Exports OpenAPI specifications from CloudFront URLs and converts them to DocumentationDocument instances. +/// Converts OpenAPI specs into search documents from the version-index catalog. /// public partial class OpenApiDocumentExporter( VersionsConfiguration versionsConfiguration, - IDocumentInferrerService? documentInferrer = null) + IDocumentInferrerService? documentInferrer = null, + VersionIndexClient? versionIndexClient = null, + IOpenApiSpecificationReader? openApiReader = null, + IDiagnosticsCollector? collector = null) { - private static readonly HttpClient HttpClient = new(); - - private const string ElasticsearchOpenApiUrl = "https://d31bhlox0wglh.cloudfront.net/elasticsearch-openapi-docs.json"; - private const string KibanaOpenApiUrl = "https://d31bhlox0wglh.cloudfront.net/kibana-openapi.json"; + private readonly IOpenApiSpecificationReader _openApiReader = openApiReader ?? OpenApiReader.Instance; + private readonly IDiagnosticsCollector _collector = collector ?? new DiagnosticsCollector([]); [GeneratedRegex(@"Added in (\d+\.\d+\.\d+)", RegexOptions.IgnoreCase)] private static partial Regex AddedInVersionRegex(); @@ -40,80 +39,105 @@ public partial class OpenApiDocumentExporter( private static partial Regex OperationVerbPathRegex(); /// - /// Fetches and processes both Elasticsearch and Kibana OpenAPI specifications. + /// Resolves every version of each configured API from the version index and converts + /// operations to search documents. Used when assembler-api-explorer is enabled. /// - /// Optional limit of documents to return per source (Elasticsearch and Kibana) - /// Cancellation token - /// Enumerable of DocumentationDocument instances for all endpoints - public async IAsyncEnumerable ExportDocuments(int? limitPerSource = null, [EnumeratorCancellation] Cancel ctx = default) + public async IAsyncEnumerable ExportDocuments( + IReadOnlyList sources, + [EnumeratorCancellation] Cancel ctx = default) { - // Process Elasticsearch API - var elasticsearchCount = 0; - await foreach (var doc in ExportFromUrl(ElasticsearchOpenApiUrl, "elasticsearch", ctx)) + VersionIndexClient? ownedClient = null; + var client = versionIndexClient ?? (ownedClient = new VersionIndexClient()); + try { - yield return doc; - elasticsearchCount++; - if (limitPerSource.HasValue && elasticsearchCount >= limitPerSource.Value) - break; + foreach (var source in sources) + { + await foreach (var doc in ExportSource(client, source, ctx).ConfigureAwait(false)) + yield return doc; + } } - - // Process Kibana API - var kibanaCount = 0; - await foreach (var doc in ExportFromUrl(KibanaOpenApiUrl, "kibana", ctx)) + finally { - yield return doc; - kibanaCount++; - if (limitPerSource.HasValue && kibanaCount >= limitPerSource.Value) - break; + ownedClient?.Dispose(); } } - /// - /// Fetches OpenAPI spec from a URL and converts it to DocumentationDocument instances. - /// - private async IAsyncEnumerable ExportFromUrl( - string url, - string product, + private async IAsyncEnumerable ExportSource( + VersionIndexClient client, + OpenApiExportSource source, [EnumeratorCancellation] Cancel ctx) { - var openApiDocument = await FetchOpenApiDocument(url, ctx); - if (openApiDocument == null) - yield break; + var versionless = source.ApiConfig.Product.VersioningSystem?.IsVersionless == true; + var versions = await client.ResolveVersionsAsync(source.Git, source.ApiKey, source.ApiConfig, _collector, ctx) + .ConfigureAwait(false); + var versionsToExport = versionless + ? versions.Where(v => v.Moniker == "main") + : versions; + + foreach (var version in versionsToExport) + { + var document = await ReadVersionDocument(client, source, version, ctx).ConfigureAwait(false); + if (document is null) + continue; - foreach (var doc in ConvertToDocuments(openApiDocument, product)) - yield return doc; + foreach (var doc in ConvertToDocuments(document, CreateConvertContext(source, version))) + yield return doc; + } } - /// - /// Fetches and parses an OpenAPI document from a URL. - /// - private static async Task FetchOpenApiDocument(string url, Cancel ctx) + private async Task ReadVersionDocument( + VersionIndexClient client, + OpenApiExportSource source, + ResolvedApiVersion version, + Cancel ctx) { - try - { - var response = await HttpClient.GetAsync(url, ctx); - _ = response.EnsureSuccessStatusCode(); + if (version.IsLocal) + return await _openApiReader.ReadAsync(version.LocalFile!).ConfigureAwait(false); - await using var stream = await response.Content.ReadAsStreamAsync(ctx); - var settings = new OpenApiReaderSettings { LeaveStreamOpen = false, RuleSet = ValidationRuleSet.GetEmptyRuleSet() }; - var openApiDocument = await OpenApiDocument.LoadAsync(stream, settings: settings, cancellationToken: ctx); - - return openApiDocument.Document; - } - catch (Exception ex) - { - Console.Error.WriteLine($"Failed to fetch OpenAPI document from {url}: {ex.Message}"); + var stream = await client.FetchSpecStreamAsync(source.ApiKey, version, _collector, ctx).ConfigureAwait(false); + if (stream is null) return null; - } + + return await _openApiReader.ReadAsync(stream, source.ApiConfig.SpecFileName).ConfigureAwait(false); + } + + private OpenApiConvertContext CreateConvertContext(OpenApiExportSource source, ResolvedApiVersion version) + { + var current = source.ApiConfig.Product.VersioningSystem?.Current + ?? versionsConfiguration.GetVersioningSystem(VersioningSystemId.Stack).Current; + var ceiling = version.Moniker == "main" + ? current + : ParseFilterCeiling(version.Version, current); + return new OpenApiConvertContext( + source.ApiKey, + version.Moniker, + ceiling, + source.ApiConfig.Product.DisplayName, + source.ApiConfig.Product.Id); + } + + internal static SemVersion ParseFilterCeiling(string version, SemVersion fallback) + { + if (SemVersion.TryParse(version, out var parsed)) + return parsed; + if (SemVersion.TryParse(version + ".0", out parsed)) + return parsed; + return fallback; } /// /// Converts an OpenAPI document to DocumentationDocument instances. /// Internal (rather than private) so tests can exercise it against an in-memory spec. /// - internal IEnumerable ConvertToDocuments(OpenApiDocument openApiDocument, string product) + internal IEnumerable ConvertToDocuments(OpenApiDocument openApiDocument, OpenApiConvertContext convert) { - var productUrl = ApiUrlBuilder.ProductRoot("/docs", product); + var productUrl = ApiUrlBuilder.ProductRoot("/docs", ApiUrlBuilder.ProductSuffix(convert.ApiKey, convert.VersionMoniker)); + var productLabel = convert.VersionMoniker == "main" + ? $"{convert.DisplayName} API" + : $"{convert.DisplayName} {convert.VersionMoniker}.x API"; + var inference = documentInferrer?.InferForOpenApi(convert.ProductId); + + yield return CreateProductLanding(openApiDocument, productUrl, productLabel, inference); foreach (var path in openApiDocument.Paths) { @@ -124,25 +148,17 @@ internal IEnumerable ConvertToDocuments(OpenApiDocument o { var operationId = operation.Value.OperationId ?? GenerateOperationId(operation.Key, path.Key); - // Check x-state extension for version filtering - if (!ShouldIncludeOperation(operation.Value)) + if (!ShouldIncludeOperation(operation.Value, convert.FilterCeiling)) continue; var operationMoniker = ApiUrlBuilder.OperationMoniker(operationId, path.Key); var url = $"{productUrl}/operation/{operationMoniker}"; - var productName = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(product); - // Trim: spec summaries occasionally carry stray leading/trailing whitespace or a - // trailing newline, which would otherwise flow verbatim into the indexed title. var summary = operation.Value.Summary?.Trim(); - // inject product name into title to ensure differentiation and better scoring - var title = $"{(string.IsNullOrEmpty(summary) ? operationId : summary)} - {productName} API"; - // append the raw operation id (e.g. "_bulk") so the REST endpoint name is searchable — - // keep it verbatim (no case/underscore normalization) since that's exactly what users type. + var title = $"{(string.IsNullOrEmpty(summary) ? operationId : summary)} - {productLabel}"; var searchTitle = $"{title} - {operationId}"; var description = TransformOperationListToMarkdown(operation.Value.Description); - // Build body content from operation details var bodyBuilder = new StringBuilder(); _ = bodyBuilder.AppendLine($"# {title}"); _ = bodyBuilder.AppendLine(); @@ -157,7 +173,6 @@ internal IEnumerable ConvertToDocuments(OpenApiDocument o _ = bodyBuilder.AppendLine($"**Path:** {path.Key}"); _ = bodyBuilder.AppendLine(); - // Add parameters if any if (operation.Value.Parameters?.Count > 0) { _ = bodyBuilder.AppendLine("## Parameters"); @@ -168,19 +183,14 @@ internal IEnumerable ConvertToDocuments(OpenApiDocument o var body = bodyBuilder.ToString(); - // Extract tags as headings var headings = operation.Value.Tags? .Select(t => t.Name) .Where(n => !string.IsNullOrEmpty(n)) .OfType() .ToArray() ?? []; - // Extract ApplicableTo from x-state var applies = ExtractApplicableTo(operation.Value); - // Infer product and repository metadata - var inference = documentInferrer?.InferForOpenApi(product); - yield return new DocumentationDocument { ContentType = "api", @@ -195,7 +205,7 @@ internal IEnumerable ConvertToDocuments(OpenApiDocument o Parents = [ new ParentDocument { Title = "API Reference", Path = "/docs/api" }, - new ParentDocument { Title = product, Path = productUrl } + new ParentDocument { Title = convert.DisplayName, Path = productUrl } ], Product = inference?.Product?.Id, RelatedProducts = inference?.RelatedProducts.Count > 0 @@ -210,38 +220,65 @@ internal IEnumerable ConvertToDocuments(OpenApiDocument o } } - /// - /// Determines if an operation should be included based on its x-state extension. - /// - private bool ShouldIncludeOperation(OpenApiOperation operation) + private static DocumentationDocument CreateProductLanding( + OpenApiDocument openApiDocument, + string productUrl, + string productLabel, + DocumentInferenceResult? inference) + { + var bodyBuilder = new StringBuilder(); + _ = bodyBuilder.AppendLine($"# {productLabel}"); + var description = openApiDocument.Info?.Description; + if (!string.IsNullOrEmpty(description)) + { + _ = bodyBuilder.AppendLine(); + _ = bodyBuilder.AppendLine(description); + } + + return new DocumentationDocument + { + ContentType = "api", + Path = productUrl, + Title = productLabel, + SearchTitle = productLabel, + Body = bodyBuilder.ToString(), + Links = [], + Parents = + [ + new ParentDocument { Title = "API Reference", Path = "/docs/api" } + ], + Product = inference?.Product?.Id, + RelatedProducts = inference?.RelatedProducts.Count > 0 + ? inference.RelatedProducts.Select(p => new IndexedProduct + { + Id = p.Id, + Repository = p.Repository ?? inference.Repository + }).ToArray() + : null + }; + } + + private static bool ShouldIncludeOperation(OpenApiOperation operation, SemVersion filterCeiling) { - // Try to get x-state extension if (operation.Extensions == null || !operation.Extensions.TryGetValue("x-state", out var stateExtension)) - return true; // No x-state, safe to include + return true; - // Get the state string value from JsonNodeExtension if (stateExtension is not JsonNodeExtension jsonNodeExtension) - return true; // Not a JSON node, safe to include + return true; var stateValue = jsonNodeExtension.Node.GetValue(); if (string.IsNullOrEmpty(stateValue)) - return true; // Empty state, safe to include + return true; - // Parse version from "Added in X.Y.Z" var match = AddedInVersionRegex().Match(stateValue); if (!match.Success) - return true; // No version found, safe to include + return true; var versionString = match.Groups[1].Value; if (!SemVersion.TryParse(versionString, out var addedInVersion)) - return true; // Could not parse version, safe to include - - // All API products currently version against Stack - var versioningSystem = versionsConfiguration.GetVersioningSystem(VersioningSystemId.Stack); - var currentVersion = versioningSystem.Current; + return true; - // Include if added version is <= current version - return addedInVersion <= currentVersion; + return addedInVersion <= filterCeiling; } /// diff --git a/src/Elastic.ApiExplorer/Export/OpenApiExportSource.cs b/src/Elastic.ApiExplorer/Export/OpenApiExportSource.cs new file mode 100644 index 0000000000..c801f477a2 --- /dev/null +++ b/src/Elastic.ApiExplorer/Export/OpenApiExportSource.cs @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation; +using Elastic.Documentation.Configuration.Toc; +using Elastic.Documentation.Versions; + +namespace Elastic.ApiExplorer.Export; + +/// +/// One api: product to index from the version index, with the git checkout used to +/// resolve repository: when the config entry does not override it. +/// +public sealed record OpenApiExportSource( + string ApiKey, + ResolvedApiConfiguration ApiConfig, + GitCheckoutInformation Git); + +/// +/// Conversion inputs for one OpenAPI spec version. is the URL moniker; +/// is the products.yml id used for inference. +/// +internal readonly record struct OpenApiConvertContext( + string ApiKey, + string VersionMoniker, + SemVersion FilterCeiling, + string DisplayName, + string ProductId); diff --git a/src/Elastic.ApiExplorer/OpenApiGenerator.cs b/src/Elastic.ApiExplorer/OpenApiGenerator.cs index 1508f00e69..6e5994cbeb 100644 --- a/src/Elastic.ApiExplorer/OpenApiGenerator.cs +++ b/src/Elastic.ApiExplorer/OpenApiGenerator.cs @@ -44,6 +44,9 @@ public class OpenApiGenerator( private readonly StaticFileContentHashProvider _contentHashProvider = new(new EmbeddedOrPhysicalFileProvider(context)); private readonly VersionIndexClient _versionIndexClient = versionIndexClient ?? new VersionIndexClient(); private readonly IOpenApiSpecificationReader _openApiReader = openApiReader ?? OpenApiReader.Instance; + private readonly List _generatedPageUrls = []; + + public IReadOnlyList GeneratedPageUrls => _generatedPageUrls; public LandingNavigationItem CreateNavigation(string apiUrlSuffix, OpenApiDocument openApiDocument, ResolvedApiConfiguration? apiConfig = null) => new ApiNavigationBuilder(_logger, context).CreateNavigation(apiUrlSuffix, openApiDocument, apiConfig); @@ -283,6 +286,7 @@ private async Task Render(INavigationItem current, T page, ApiRend }; await using var stream = _writeFileSystem.FileStream.New(outputFile.FullName, FileMode.OpenOrCreate); await page.RenderAsync(stream, renderContext, ctx); + _generatedPageUrls.Add(current.Url); return outputFile; IFileInfo OutputFile(INavigationItem currentNavigation) diff --git a/src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.Export.cs b/src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.Export.cs index 518abda663..0521e642d0 100644 --- a/src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.Export.cs +++ b/src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.Export.cs @@ -22,6 +22,10 @@ namespace Elastic.Markdown.Exporters.Elasticsearch; public partial class ElasticsearchMarkdownExporter { private IDocumentInferrerService? _inferService; + private IReadOnlyList _openApiSources = []; + + public void ConfigureOpenApiExport(IReadOnlyList sources) => + _openApiSources = sources; /// /// Assigns hash, last updated, and batch index date to a documentation document. @@ -94,6 +98,10 @@ internal static void CommonEnrichments(DocumentationDocument doc, INavigationIte doc.SearchTitle = CreateSearchTitle(); // if we have no navigation, initialize to 20 since rank_feature would score 0 too high doc.Navigation.Depth = navigationItem?.NavigationDepth ?? 20; + if (doc.ContentType == "api" && IsVersionedApiPath(doc.Path)) + doc.Navigation.Depth = 40; + if (doc.ContentType == "api" && IsProductLandingPath(doc.Path)) + doc.Navigation.Depth = IsVersionedApiPath(doc.Path) ? 30 : 10; doc.Navigation.TableOfContents = navigationItem switch { // release-notes get effectively flattened by product, so we to dampen its effect slightly @@ -119,6 +127,32 @@ internal static void CommonEnrichments(DocumentationDocument doc, INavigationIte // OpenApiDocumentExporter actually sets to "api".) doc.ContentTier = doc.ContentType == "api" ? ContentTiers.Reference : ClassifyContentTier(navigationItem, doc.Path); + static bool IsVersionedApiPath(string path) + { + var parts = path.Split('/', RemoveEmptyEntries); + return parts.Length >= 5 + && parts[0] == "docs" + && parts[1] == "api" + && parts[2] == "doc" + && parts[4].Length > 1 + && parts[4][0] == 'v' + && char.IsDigit(parts[4][1]); + } + + static bool IsProductLandingPath(string path) + { + var parts = path.Split('/', RemoveEmptyEntries); + if (parts.Length is not (4 or 5)) + return false; + if (parts[0] != "docs" || parts[1] != "api" || parts[2] != "doc") + return false; + if (parts.Length == 4) + return true; + return parts[4].Length > 1 + && parts[4][0] == 'v' + && char.IsDigit(parts[4][1]); + } + string CreateSearchTitle() { // skip doc and the section @@ -236,13 +270,16 @@ public async ValueTask FinishExportAsync(IDirectoryInfo outputFolder, Canc return true; } - // this is temporary; once we implement Elastic.ApiExplorer, this should flow through - // we'll rename IMarkdownExporter to IDocumentationFileExporter at that point - _logger.LogInformation("Exporting OpenAPI documentation to Elasticsearch"); + if (_openApiSources.Count == 0) + { + _logger.LogInformation("Skipping OpenAPI export: assembler-api-explorer is disabled or no API sources were discovered"); + return true; + } - var exporter = new OpenApiDocumentExporter(_versionsConfiguration, _inferService); + _logger.LogInformation("Exporting OpenAPI documentation to Elasticsearch"); - await foreach (var doc in exporter.ExportDocuments(limitPerSource: null, ctx)) + var exporter = new OpenApiDocumentExporter(_versionsConfiguration, _inferService, collector: _collector); + await foreach (var doc in exporter.ExportDocuments(_openApiSources, ctx)) { var document = MarkdownParser.Parse(doc.Body ?? string.Empty); diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs index 5e4b1544f0..3537594fa8 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuildService.cs @@ -134,20 +134,20 @@ Cancel ctx if (exporters.Contains(Exporter.Html)) { + var features = assembleContext.Environment.ToFeatureFlags(); var openApiStopwatch = Stopwatch.StartNew(); - await AssemblerOpenApiBuildStep.BuildAsync(logFactory, assembleContext, assembleSources, ctx); + var apiUrls = await AssemblerOpenApiBuildStep.BuildAsync(logFactory, assembleContext, assembleSources, ctx); openApiStopwatch.Stop(); _logger.LogInformation("OpenAPI build step completed in {DurationMs} ms", openApiStopwatch.ElapsedMilliseconds); - // Build-time sitemap uses current date as placeholder for backwards compatibility. - // Production sitemap with correct content_last_updated dates is generated via - // `assembler sitemap` after ES indexing, which overwrites this file. var urls = navigation.NavigationItems .SelectMany(SitemapNavigationHelper.Flatten) .Select(n => n.Url) .Distinct(); var now = DateTimeOffset.UtcNow; var entries = urls.ToDictionary(u => u, _ => now); + foreach (var apiUrl in apiUrls) + _ = entries.TryAdd(apiUrl, now); if (entries.Count >= SitemapBuilder.WarningEntryThreshold) collector.EmitGlobalWarning( @@ -155,7 +155,11 @@ Cancel ctx "Consider implementing sitemap index files." ); - var sitemapResult = SitemapBuilder.Generate(entries, assembleContext.WriteFileSystem, assembleContext.OutputWithPathPrefixDirectory); + var sitemapResult = SitemapBuilder.Generate( + entries, + assembleContext.WriteFileSystem, + assembleContext.OutputWithPathPrefixDirectory, + includeApiDocs: features.AssemblerApiExplorerEnabled); if (sitemapResult.FileSizeBytes >= SitemapBuilder.WarningFileSizeBytes) collector.EmitGlobalWarning( diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs index 4a7c0d61db..7e116c9b24 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs @@ -17,6 +17,7 @@ using Elastic.Documentation.Serialization; using Elastic.Markdown; using Elastic.Markdown.Exporters; +using Elastic.Markdown.Exporters.Elasticsearch; using Microsoft.Extensions.Logging; namespace Elastic.Documentation.Assembler.Building; @@ -49,6 +50,7 @@ public async Task BuildAllAsync(FrozenDictionary await e.StartAsync(ctx)); await Task.WhenAll(tasks); @@ -116,6 +118,18 @@ public async Task BuildAllAsync(FrozenDictionary markdownExporters, + FrozenDictionary assembleSets) + { + if (!context.Environment.ToFeatureFlags().AssemblerApiExplorerEnabled) + return; + + var sources = AssemblerOpenApiBuildStep.DiscoverExportSources(assembleSets, context.Collector); + foreach (var exporter in markdownExporters.OfType()) + exporter.ConfigureOpenApiExport(sources); + } + private void CollectRedirects( Dictionary allRedirects, IReadOnlyDictionary redirects, diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs index 5908c4b4aa..d47c505ff8 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerOpenApiBuildStep.cs @@ -5,11 +5,11 @@ using System.Collections.Frozen; using System.Diagnostics; using Elastic.ApiExplorer; +using Elastic.ApiExplorer.Export; using Elastic.ApiExplorer.Landing; using Elastic.ApiExplorer.Model; using Elastic.Documentation; using Elastic.Documentation.Assembler.Navigation; -using Elastic.Documentation.Configuration.Builder; using Elastic.Documentation.Diagnostics; using Elastic.Markdown; using Microsoft.Extensions.Logging; @@ -21,7 +21,7 @@ namespace Elastic.Documentation.Assembler.Building; /// public static class AssemblerOpenApiBuildStep { - public static async Task BuildAsync( + public static async Task> BuildAsync( ILoggerFactory logFactory, AssembleContext assembleContext, AssembleSources assembleSources, @@ -29,25 +29,24 @@ public static async Task BuildAsync( { var logger = logFactory.CreateLogger(typeof(AssemblerOpenApiBuildStep)); var env = assembleContext.Environment; - var features = new FeatureFlags([]); - foreach (var (key, value) in env.FeatureFlags) - features.Set(key, value); + var features = env.ToFeatureFlags(); if (!features.AssemblerApiExplorerEnabled) { logger.LogInformation("Skipping OpenAPI generation: assembler-api-explorer feature flag is disabled"); - return; + return []; } var owners = DiscoverApiOwners(assembleSources.AssembleSets, assembleContext.Collector); if (owners.Count == 0) { logger.LogInformation("Skipping OpenAPI generation: no API declarations found in assembled docsets"); - return; + return []; } var stopwatch = Stopwatch.StartNew(); var catalogEntries = new List(); + var generatedUrls = new List(); using var versionIndexClient = new VersionIndexClient(); foreach (var owner in owners) @@ -61,6 +60,7 @@ public static async Task BuildAsync( versionIndexClient); var entries = await openApiGenerator.GenerateProducts(ctx).ConfigureAwait(false); catalogEntries.AddRange(entries); + generatedUrls.AddRange(openApiGenerator.GeneratedPageUrls); } if (catalogEntries.Count > 0) @@ -72,6 +72,7 @@ public static async Task BuildAsync( new DocumentationGenerator(owners[0].Set.DocumentationSet, logFactory).MarkdownStringRenderer, versionIndexClient); await catalogGenerator.GenerateCatalog(catalogEntries, ctx).ConfigureAwait(false); + generatedUrls.AddRange(catalogGenerator.GeneratedPageUrls); } stopwatch.Stop(); @@ -79,6 +80,25 @@ public static async Task BuildAsync( "Finished generating OpenAPI pages under {OutputDirectory} in {DurationMs} ms", assembleContext.OutputWithPathPrefixDirectory.FullName, stopwatch.ElapsedMilliseconds); + return generatedUrls; + } + + public static IReadOnlyList DiscoverExportSources( + FrozenDictionary assembleSets, + IDiagnosticsCollector collector) + { + var sources = new List(); + foreach (var owner in DiscoverApiOwners(assembleSets, collector)) + { + var apiConfigurations = owner.Set.BuildContext.Configuration.ApiConfigurations; + if (apiConfigurations is null) + continue; + + foreach (var (apiKey, apiConfig) in apiConfigurations) + sources.Add(new OpenApiExportSource(apiKey, apiConfig, owner.Set.BuildContext.Git)); + } + + return sources; } internal static IReadOnlyList DiscoverApiOwners( diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerSitemapService.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerSitemapService.cs index 7542c7bfd5..59298d76ac 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerSitemapService.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerSitemapService.cs @@ -74,7 +74,11 @@ public async Task GenerateSitemapAsync( "Consider implementing sitemap index files." ); - var result = SitemapBuilder.Generate(entries, assembleContext.WriteFileSystem, assembleContext.OutputWithPathPrefixDirectory); + var result = SitemapBuilder.Generate( + entries, + assembleContext.WriteFileSystem, + assembleContext.OutputWithPathPrefixDirectory, + includeApiDocs: assembleContext.Environment.ToFeatureFlags().AssemblerApiExplorerEnabled); if (result.FileSizeBytes >= SitemapBuilder.WarningFileSizeBytes) collector.EmitGlobalWarning( diff --git a/src/services/Elastic.Documentation.Assembler/Building/SitemapBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/SitemapBuilder.cs index 06130da5be..9cbc23fdcd 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/SitemapBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/SitemapBuilder.cs @@ -27,14 +27,13 @@ public static class SitemapBuilder public static SitemapResult Generate( IReadOnlyDictionary entries, IFileSystem fileSystem, - IDirectoryInfo outputFolder + IDirectoryInfo outputFolder, + bool includeApiDocs = false ) { - // API pages are generated only on staging (assembler-api-explorer flag) and /docs/api/* is still - // proxied to bump.sh at the edge (#725). Keep them out of the sitemap until cutover. - var filtered = entries - .Where(e => !e.Key.StartsWith("/docs/api/", StringComparison.Ordinal)) - .ToList(); + var filtered = includeApiDocs + ? entries.ToList() + : entries.Where(e => !e.Key.StartsWith("/docs/api/", StringComparison.Ordinal)).ToList(); if (filtered.Count > MaxEntries) throw new InvalidOperationException( diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterTests.cs index 21e07ad659..ceff8f4fd0 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterTests.cs @@ -2,175 +2,176 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.Collections.Concurrent; +using System.Text.Json.Nodes; using AwesomeAssertions; using Elastic.ApiExplorer.Export; using Elastic.ApiExplorer.Model; -using Elastic.ApiExplorer.Operations; -using Elastic.Documentation; using Elastic.Documentation.Configuration.Versions; -using Elastic.Documentation.Search; using Elastic.Documentation.Search.Contract; using Elastic.Documentation.Versions; +using Microsoft.OpenApi; using static System.StringComparison; namespace Elastic.ApiExplorer.Tests; public class OpenApiDocumentExporterTests { - private static readonly HttpClient HttpClient = new(); - private const string BaseUrl = "https://www.elastic.co"; + private static readonly VersionsConfiguration VersionsConfiguration = + TestHelpers.CreateStackVersionsConfiguration(currentMajor: 9, currentMinor: 2); - [Fact(Skip = "This spams elastic.co, run this manually")] - public async Task ExportedDocumentUrlsShouldReturnSuccessStatusCode() + private static OpenApiConvertContext ElasticsearchContext(string moniker = "main", SemVersion? ceiling = null) => + new("elasticsearch", moniker, ceiling ?? new SemVersion(9, 2, 0), "Elasticsearch", "elasticsearch"); + + private static OpenApiDocument PingSpec(string? xState = null, string? description = null) { - // Arrange - var versionsConfiguration = new VersionsConfiguration + var operation = new OpenApiOperation + { + OperationId = "ping", + Summary = "Ping", + Description = description + }; + if (xState is not null) + operation.Extensions = new Dictionary { ["x-state"] = new JsonNodeExtension(JsonValue.Create(xState)) }; + + return new OpenApiDocument { - VersioningSystems = new Dictionary + Paths = new OpenApiPaths { + ["/ping"] = new OpenApiPathItem { - VersioningSystemId.Stack, - new VersioningSystem + Operations = new Dictionary { - Id = VersioningSystemId.Stack, - Base = new SemVersion(8, 0, 0), - Current = new SemVersion(9, 2, 0) + [HttpMethod.Get] = operation } } } }; + } - var exporter = new OpenApiDocumentExporter(versionsConfiguration); - const int limitPerSource = 300; // Get 50 from each source (Elasticsearch and Kibana) + [Fact] + public void ConvertToDocuments_Main_UsesUnversionedOperationUrl() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); - // Act - Collect all documents, tracking source - var documents = new List<(string Url, string Source)>(); - await foreach (var doc in exporter.ExportDocuments(limitPerSource, TestContext.Current.CancellationToken)) - { - if (!string.IsNullOrEmpty(doc.Path)) - { - // Determine source from URL - var source = doc.Path.Contains("/elasticsearch/") ? "elasticsearch" : "kibana"; - documents.Add((doc.Path, source)); - } - } + var docs = exporter.ConvertToDocuments(PingSpec(), ElasticsearchContext()).ToArray(); + var operations = OperationDocs(docs); - // Assert we have documents from both sources - documents.Should().NotBeEmpty("the exporter should return at least some documents"); - var elasticsearchDocs = documents.Where(d => d.Source == "elasticsearch").ToList(); - var kibanaDocs = documents.Where(d => d.Source == "kibana").ToList(); + operations.Should().ContainSingle(); + operations[0].Path.Should().Be("/docs/api/doc/elasticsearch/operation/operation-ping"); + operations[0].Title.Should().Be("Ping - Elasticsearch API"); + operations[0].Parents.Should().Contain(p => p.Path == "/docs/api/doc/elasticsearch"); + } - elasticsearchDocs.Should().NotBeEmpty("should have Elasticsearch documents"); - kibanaDocs.Should().NotBeEmpty("should have Kibana documents"); + [Fact] + public void ConvertToDocuments_NumericMoniker_UsesVersionPrefixedUrlAndTitle() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); - // Take all documents as sample (already limited) - var sample = documents.Select(d => d.Url).ToList(); + var docs = exporter.ConvertToDocuments(PingSpec(), ElasticsearchContext("8", new SemVersion(8, 19, 0))).ToArray(); + var operations = OperationDocs(docs); - // Test each URL in parallel - var failures = new ConcurrentBag<(string Url, int StatusCode)>(); + operations.Should().ContainSingle(); + operations[0].Path.Should().Be("/docs/api/doc/elasticsearch/v8/operation/operation-ping"); + operations[0].Title.Should().Be("Ping - Elasticsearch 8.x API"); + operations[0].Parents.Should().Contain(p => p.Path == "/docs/api/doc/elasticsearch/v8"); + } - await Parallel.ForEachAsync(sample, - new ParallelOptions { MaxDegreeOfParallelism = 10, CancellationToken = TestContext.Current.CancellationToken }, - async (url, ct) => - { - var fullUrl = $"{BaseUrl}{url}"; + [Fact] + public void ConvertToDocuments_AddedInAfterCeiling_IsExcluded() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); + var spec = PingSpec("Generally available; Added in 8.19.0"); - try - { - using var request = new HttpRequestMessage(HttpMethod.Head, fullUrl); - - // Mimic browser headers - request.Headers.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); - request.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"); - request.Headers.Add("Accept-Language", "en-US,en;q=0.9"); - request.Headers.Add("Accept-Encoding", "gzip, deflate, br"); - request.Headers.Add("DNT", "1"); - request.Headers.Add("Connection", "keep-alive"); - request.Headers.Add("Upgrade-Insecure-Requests", "1"); - request.Headers.Add("Sec-Fetch-Dest", "document"); - request.Headers.Add("Sec-Fetch-Mode", "navigate"); - request.Headers.Add("Sec-Fetch-Site", "none"); - request.Headers.Add("Sec-Fetch-User", "?1"); - request.Headers.Add("Cache-Control", "max-age=0"); - - var response = await HttpClient.SendAsync( - request, - HttpCompletionOption.ResponseHeadersRead, - ct - ); - - if (!response.IsSuccessStatusCode) - { - failures.Add((url, (int)response.StatusCode)); - } - } - catch - { - failures.Add((url, -1)); // Use -1 to indicate exception - } - }); + var docs = exporter.ConvertToDocuments(spec, ElasticsearchContext("8", new SemVersion(8, 18, 0))).ToArray(); - // Assert all URLs returned 200 - failures.Should().BeEmpty( - $"all sampled URLs should return 200 OK, but the following failed: {string.Join(", ", failures.Select(f => $"{f.Url} ({f.StatusCode})"))}" - ); + OperationDocs(docs).Should().BeEmpty(); } [Fact] - public async Task DescriptionWithHtmlOperationsListShouldTransformToMarkdownAtEnd() + public void ParseFilterCeiling_MajorMinor_AppendsPatchZero() { - // Arrange - var versionsConfiguration = new VersionsConfiguration - { - VersioningSystems = new Dictionary - { - { - VersioningSystemId.Stack, - new VersioningSystem - { - Id = VersioningSystemId.Stack, - Base = new SemVersion(8, 0, 0), - Current = new SemVersion(9, 2, 0) - } - } - } - }; + var fallback = new SemVersion(9, 2, 0); - var exporter = new OpenApiDocumentExporter(versionsConfiguration); + OpenApiDocumentExporter.ParseFilterCeiling("8.19", fallback).Should().Be(new SemVersion(8, 19, 0)); + OpenApiDocumentExporter.ParseFilterCeiling("8.19.0", fallback).Should().Be(new SemVersion(8, 19, 0)); + } - // Act - Get some Elasticsearch documents - var documents = new List(); - await foreach (var doc in exporter.ExportDocuments(limitPerSource: 100, TestContext.Current.CancellationToken)) - { - if (doc.Description != null && doc.Description.Contains("**All methods and paths for this operation:**")) - { - documents.Add(doc); - } - } + [Fact] + public void ConvertToDocuments_AddedInAtCeiling_IsIncluded() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); + var spec = PingSpec("Generally available; Added in 8.19.0"); - // Assert we found at least one document with the pattern - documents.Should().NotBeEmpty("there should be at least one document with operation list"); + var docs = exporter.ConvertToDocuments(spec, ElasticsearchContext("8", new SemVersion(8, 19, 0))).ToArray(); - foreach (var doc in documents) - { - // Should not contain HTML - doc.Description.Should().NotContain("
", "HTML should be converted to markdown"); - doc.Description.Should().NotContain(" !string.IsNullOrWhiteSpace(l)).TakeLast(5).ToList(); - - // At least one of the last few lines should be a Markdown list item - var hasMarkdownListAtEnd = lastNonEmptyLines.Any(l => l.StartsWith("- **", InvariantCulture)); - hasMarkdownListAtEnd.Should().BeTrue( - $"markdown list should be at the end of the description. Last lines:\n{string.Join("\n", lastNonEmptyLines)}\n\nFull description:\n{doc.Description}" - ); - } + OperationDocs(docs).Should().ContainSingle(); + } + + [Fact] + public void ConvertToDocuments_Main_EmitsProductLanding() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); + + var docs = exporter.ConvertToDocuments(PingSpec(), ElasticsearchContext()).ToArray(); + + docs[0].Path.Should().Be("/docs/api/doc/elasticsearch"); + docs[0].Title.Should().Be("Elasticsearch API"); + docs[0].SearchTitle.Should().Be("Elasticsearch API"); + docs[0].Parents.Should().Equal([new ParentDocument { Title = "API Reference", Path = "/docs/api" }]); + } + + [Fact] + public void ConvertToDocuments_NumericMoniker_EmitsVersionedLanding() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); + + var docs = exporter.ConvertToDocuments(PingSpec(), ElasticsearchContext("8", new SemVersion(8, 19, 0))).ToArray(); + + docs[0].Path.Should().Be("/docs/api/doc/elasticsearch/v8"); + docs[0].Title.Should().Be("Elasticsearch 8.x API"); } + + [Fact] + public void ConvertToDocuments_AddedInAfterCeiling_StillEmitsLanding() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); + var spec = PingSpec("Generally available; Added in 8.19.0"); + + var docs = exporter.ConvertToDocuments(spec, ElasticsearchContext("8", new SemVersion(8, 18, 0))).ToArray(); + + docs.Should().ContainSingle(); + docs[0].Path.Should().Be("/docs/api/doc/elasticsearch/v8"); + docs[0].Title.Should().Be("Elasticsearch 8.x API"); + OperationDocs(docs).Should().BeEmpty(); + } + + [Fact] + public void DescriptionWithHtmlOperationsListShouldTransformToMarkdownAtEnd() + { + var exporter = new OpenApiDocumentExporter(VersionsConfiguration); + var description = """ + **All methods and paths for this operation:** +
+ GET /_ping +
+ """; + var spec = PingSpec(description: description); + + var docs = exporter.ConvertToDocuments(spec, ElasticsearchContext()).ToArray(); + + var operations = OperationDocs(docs); + operations.Should().ContainSingle(); + var doc = operations[0]; + doc.Description.Should().NotContain("
"); + doc.Description.Should().NotContain(" !string.IsNullOrWhiteSpace(l)) + .TakeLast(5) + .ToList(); + lastNonEmptyLines.Any(l => l.StartsWith("- **", InvariantCulture)).Should().BeTrue(); + } + + private static DocumentationDocument[] OperationDocs(IEnumerable docs) => + docs.Where(d => d.Path.Contains("/operation/", Ordinal)).ToArray(); } diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterVersionIndexTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterVersionIndexTests.cs new file mode 100644 index 0000000000..a9fe1073bd --- /dev/null +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiDocumentExporterVersionIndexTests.cs @@ -0,0 +1,159 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using System.Text; +using AwesomeAssertions; +using Elastic.ApiExplorer.Export; +using Elastic.ApiExplorer.Model; +using Elastic.Documentation; +using Elastic.Documentation.Configuration.Products; +using Elastic.Documentation.Configuration.Toc; +using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Versions; +using FakeItEasy; +using Microsoft.OpenApi; + +namespace Elastic.ApiExplorer.Tests; + +public class OpenApiDocumentExporterVersionIndexTests +{ + private static readonly Uri BaseUri = new("https://cdn.example/"); + + private static readonly VersionsConfiguration StackVersions = TestHelpers.CreateStackVersionsConfiguration(currentMajor: 9, currentMinor: 2); + + private static GitCheckoutInformation GitForElasticsearch() => new() + { + Branch = "main", + Remote = "https://github.com/elastic/elasticsearch.git", + Ref = "refs/heads/main" + }; + + private static OpenApiDocument SpecWithOperation(string operationId) => new() + { + Info = new OpenApiInfo { Title = operationId, Version = "1.0" }, + Paths = new OpenApiPaths + { + ["/ping"] = new OpenApiPathItem + { + Operations = new Dictionary + { + [HttpMethod.Get] = new() { OperationId = operationId, Summary = "Ping" } + } + } + } + }; + + private static ResolvedApiConfiguration ApiConfig(Product product, string specFileName = "elasticsearch-openapi.json", string? repository = "elastic/elasticsearch") => + new() + { + ProductKey = product.Id, + Product = product, + SpecFileName = specFileName, + Repository = repository + }; + + private static HttpMessageHandler IndexHandler(string indexJson) => + new StubHandler(request => + { + if (request.RequestUri!.AbsolutePath.EndsWith("index.json", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(indexJson, Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(/*lang=json,strict*/ """{"openapi":"3.1.0","info":{"title":"Spec","version":"1.0"},"paths":{}}""", Encoding.UTF8, "application/json") + }; + }); + + [Fact] + public async Task ExportDocuments_MultiMajorIndex_EmitsMainAndVersionedPaths() + { + var product = TestHelpers.CreateProduct("elasticsearch", StackVersions.GetVersioningSystem(VersioningSystemId.Stack), "Elasticsearch"); + var handler = IndexHandler(/*lang=json,strict*/ """ + { + "elastic/elasticsearch": { + "elasticsearch-openapi.json": { + "main": { "version": "main" }, + "9": { "version": "9.4" }, + "8": { "version": "8.19" } + } + } + } + """); + using var client = new VersionIndexClient(BaseUri, handler, sleep: (_, _) => Task.CompletedTask); + var queue = new Queue( + [ + SpecWithOperation("ping-main"), + SpecWithOperation("ping-9"), + SpecWithOperation("ping-8") + ]); + var reader = A.Fake(); + A.CallTo(() => reader.ReadAsync(A._, A._)) + .ReturnsLazily(_ => Task.FromResult(queue.Dequeue())); + var exporter = new OpenApiDocumentExporter(StackVersions, versionIndexClient: client, openApiReader: reader, collector: new DiagnosticsCollector([])); + var source = new OpenApiExportSource("elasticsearch", ApiConfig(product), GitForElasticsearch()); + + var docs = new List(); + await foreach (var doc in exporter.ExportDocuments([source], TestContext.Current.CancellationToken)) + docs.Add(doc); + + docs.Select(d => d.Path).Should().BeEquivalentTo( + [ + "/docs/api/doc/elasticsearch", + "/docs/api/doc/elasticsearch/operation/operation-ping-main", + "/docs/api/doc/elasticsearch/v9", + "/docs/api/doc/elasticsearch/v9/operation/operation-ping-9", + "/docs/api/doc/elasticsearch/v8", + "/docs/api/doc/elasticsearch/v8/operation/operation-ping-8" + ]); + } + + [Fact] + public async Task ExportDocuments_VersionlessProduct_EmitsMainOnly() + { + var versionless = TestHelpers.CreateVersionlessConfiguration(); + var product = TestHelpers.CreateProduct("cloud-serverless", versionless.GetVersioningSystem(VersioningSystemId.Serverless), "Cloud Serverless"); + var handler = IndexHandler(/*lang=json,strict*/ """ + { + "elastic/serverless-api-specification": { + "elastic-cloud-serverless.yml": { + "main": { "version": "main" }, + "8": { "version": "8.19" } + } + } + } + """); + using var client = new VersionIndexClient(BaseUri, handler, sleep: (_, _) => Task.CompletedTask); + var reader = A.Fake(); + A.CallTo(() => reader.ReadAsync(A._, A._)) + .Returns(Task.FromResult(SpecWithOperation("ping"))); + var exporter = new OpenApiDocumentExporter(versionless, versionIndexClient: client, openApiReader: reader, collector: new DiagnosticsCollector([])); + var source = new OpenApiExportSource( + "cloud-serverless", + ApiConfig(product, specFileName: "elastic-cloud-serverless.yml", repository: "elastic/serverless-api-specification"), + GitForElasticsearch()); + + var docs = new List(); + await foreach (var doc in exporter.ExportDocuments([source], TestContext.Current.CancellationToken)) + docs.Add(doc); + + docs.Select(d => d.Path).Should().BeEquivalentTo( + [ + "/docs/api/doc/cloud-serverless", + "/docs/api/doc/cloud-serverless/operation/operation-ping" + ]); + } + + private sealed class StubHandler(Func responder) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(responder(request)); + } +} diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorMultiVersionTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorMultiVersionTests.cs index 7eff1c5823..a1936b87fa 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorMultiVersionTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiGeneratorMultiVersionTests.cs @@ -197,6 +197,8 @@ public async Task Generate_WritesDistinctOutputTreesForMainAndReleasedMajors() context.WriteFileSystem.File.Exists(Path.Join(outputRoot, "api", "doc", "elasticsearch", "v9", "index.html")).Should().BeTrue(); context.WriteFileSystem.File.Exists(Path.Join(outputRoot, "api", "doc", "elasticsearch", "v8", "index.html")).Should().BeTrue(); context.WriteFileSystem.File.Exists(Path.Join(outputRoot, "api", "doc", "elasticsearch", "v8", "operation", "operation-ping", "index.html")).Should().BeTrue(); + generator.GeneratedPageUrls.Should().Contain(u => u.Contains("/api/doc/elasticsearch/v8/operation/operation-ping", StringComparison.Ordinal)); + generator.GeneratedPageUrls.Should().Contain(u => u.Contains("/api/doc/elasticsearch", StringComparison.Ordinal) && !u.Contains("/v", StringComparison.Ordinal)); } private static BuildContext CreateGenerateContext( diff --git a/tests/Elastic.ApiExplorer.Tests/OpenApiOperationIdSearchTitleTests.cs b/tests/Elastic.ApiExplorer.Tests/OpenApiOperationIdSearchTitleTests.cs index 40a252bc91..1df0a32229 100644 --- a/tests/Elastic.ApiExplorer.Tests/OpenApiOperationIdSearchTitleTests.cs +++ b/tests/Elastic.ApiExplorer.Tests/OpenApiOperationIdSearchTitleTests.cs @@ -5,7 +5,6 @@ using AwesomeAssertions; using Elastic.ApiExplorer.Export; using Elastic.ApiExplorer.Model; -using Elastic.ApiExplorer.Operations; using Elastic.Documentation.Configuration.Versions; using Elastic.Documentation.Versions; using Microsoft.OpenApi; @@ -19,16 +18,11 @@ namespace Elastic.ApiExplorer.Tests; ///
public class OpenApiOperationIdSearchTitleTests { - private static readonly VersionsConfiguration VersionsConfiguration = new() - { - VersioningSystems = new Dictionary - { - { - VersioningSystemId.Stack, - new VersioningSystem { Id = VersioningSystemId.Stack, Base = new SemVersion(8, 0, 0), Current = new SemVersion(9, 2, 0) } - } - } - }; + private static readonly VersionsConfiguration VersionsConfiguration = + TestHelpers.CreateStackVersionsConfiguration(currentMajor: 9, currentMinor: 2); + + private static OpenApiConvertContext ElasticsearchMain() => + new("elasticsearch", "main", new SemVersion(9, 2, 0), "Elasticsearch", "elasticsearch"); private static OpenApiDocument CreateBulkSpec() => new() { @@ -53,10 +47,11 @@ public void BulkOperation_SearchTitleContainsTheRawOperationIdWithUnderscore() { var exporter = new OpenApiDocumentExporter(VersionsConfiguration); - var docs = exporter.ConvertToDocuments(CreateBulkSpec(), "elasticsearch").ToArray(); + var docs = exporter.ConvertToDocuments(CreateBulkSpec(), ElasticsearchMain()).ToArray(); + var operations = docs.Where(d => d.Path.Contains("/operation/", StringComparison.Ordinal)).ToArray(); - docs.Should().HaveCount(1); - var doc = docs[0]; + operations.Should().HaveCount(1); + var doc = operations[0]; doc.Title.Should().Be("Bulk index or delete documents - Elasticsearch API"); doc.SearchTitle.Should().Be("Bulk index or delete documents - Elasticsearch API - _bulk"); @@ -86,10 +81,11 @@ public void Operation_SummaryWithTrailingNewline_DoesNotLeakIntoTitleOrSearchTit { var exporter = new OpenApiDocumentExporter(VersionsConfiguration); - var docs = exporter.ConvertToDocuments(CreateSpecWithSummaryWhitespace("Bulk index or delete documents\n"), "elasticsearch").ToArray(); + var docs = exporter.ConvertToDocuments(CreateSpecWithSummaryWhitespace("Bulk index or delete documents\n"), ElasticsearchMain()).ToArray(); + var operations = docs.Where(d => d.Path.Contains("/operation/", StringComparison.Ordinal)).ToArray(); - docs.Should().HaveCount(1); - var doc = docs[0]; + operations.Should().HaveCount(1); + var doc = operations[0]; doc.Title.Should().Be("Bulk index or delete documents - Elasticsearch API"); doc.SearchTitle.Should().Be("Bulk index or delete documents - Elasticsearch API - _bulk"); @@ -102,10 +98,11 @@ public void Operation_BlankSummary_FallsBackToOperationId() { var exporter = new OpenApiDocumentExporter(VersionsConfiguration); - var docs = exporter.ConvertToDocuments(CreateSpecWithSummaryWhitespace(" "), "elasticsearch").ToArray(); + var docs = exporter.ConvertToDocuments(CreateSpecWithSummaryWhitespace(" "), ElasticsearchMain()).ToArray(); + var operations = docs.Where(d => d.Path.Contains("/operation/", StringComparison.Ordinal)).ToArray(); - docs.Should().HaveCount(1); - var doc = docs[0]; + operations.Should().HaveCount(1); + var doc = operations[0]; doc.Title.Should().Be("_bulk - Elasticsearch API"); } diff --git a/tests/Elastic.Documentation.Build.Tests/AssemblerOpenApiBuildStepTests.cs b/tests/Elastic.Documentation.Build.Tests/AssemblerOpenApiBuildStepTests.cs index f3c5d250ec..a7492dd7e4 100644 --- a/tests/Elastic.Documentation.Build.Tests/AssemblerOpenApiBuildStepTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/AssemblerOpenApiBuildStepTests.cs @@ -62,12 +62,13 @@ public async Task BuildAsync_SkipsWhenFeatureFlagDisabled() outputDirectory); var assembleSources = AssembleSources.ForTests(context, FrozenDictionary.Empty); - await AssemblerOpenApiBuildStep.BuildAsync( + var urls = await AssemblerOpenApiBuildStep.BuildAsync( NullLoggerFactory.Instance, context, assembleSources, TestContext.Current.CancellationToken); + urls.Should().BeEmpty(); fileSystem.Directory.Exists(fileSystem.Path.Join(outputDirectory, "docs", "api")) .Should().BeFalse("OpenAPI generation must not run when the feature flag is disabled"); } @@ -92,16 +93,34 @@ public async Task BuildAsync_SkipsWhenNoApiDeclarationsAndFlagEnabled() outputDirectory); var assembleSources = AssembleSources.ForTests(context, FrozenDictionary.Empty); - await AssemblerOpenApiBuildStep.BuildAsync( + var urls = await AssemblerOpenApiBuildStep.BuildAsync( NullLoggerFactory.Instance, context, assembleSources, TestContext.Current.CancellationToken); + urls.Should().BeEmpty(); fileSystem.Directory.Exists(fileSystem.Path.Join(outputDirectory, "docs", "api")) .Should().BeFalse("OpenAPI generation must not run without API declarations"); } + [Fact] + public void DiscoverExportSources_MapsApiKeysToExportSources() + { + var collector = new DiagnosticsCollector([]); + var withApi = CreateDocumentationSet("docs-content", "elasticsearch", collector); + var assembleSets = new Dictionary + { + [withApi.Checkout.Repository.Name] = withApi + }.ToFrozenDictionary(); + + var sources = AssemblerOpenApiBuildStep.DiscoverExportSources(assembleSets, collector); + + sources.Should().ContainSingle(); + sources[0].ApiKey.Should().Be("elasticsearch"); + sources[0].Git.Should().Be(withApi.BuildContext.Git); + } + [Fact] public void DiscoverApiOwners_EmitsErrorWhenDuplicateKeysDeclared() { diff --git a/tests/Elastic.Documentation.Build.Tests/SitemapTests.cs b/tests/Elastic.Documentation.Build.Tests/SitemapTests.cs index 6734381784..6ab0728e9d 100644 --- a/tests/Elastic.Documentation.Build.Tests/SitemapTests.cs +++ b/tests/Elastic.Documentation.Build.Tests/SitemapTests.cs @@ -181,6 +181,34 @@ public void Generate_ExcludesApiDocsFromSitemap() locs.Should().Contain("https://www.elastic.co/docs/kibana/dashboard"); } + [Fact] + public void Generate_IncludesApiDocs_WhenRequested() + { + var fs = new MockFileSystem(); + var outputDir = fs.DirectoryInfo.New("/output"); + var now = DateTimeOffset.UtcNow; + var entries = new Dictionary + { + ["/docs/elasticsearch/getting-started"] = now, + ["/docs/api/"] = now, + ["/docs/api/doc/elasticsearch/"] = now, + ["/docs/api/doc/elasticsearch/operation/operation-ping"] = now, + ["/docs/api/doc/elasticsearch/v8/operation/operation-ping"] = now, + }; + + SitemapBuilder.Generate(entries, fs, outputDir, includeApiDocs: true); + + var content = fs.File.ReadAllText(fs.Path.Join("/output", "sitemap.xml")); + var doc = XDocument.Parse(content); + XNamespace ns = "http://www.sitemaps.org/schemas/sitemap/0.9"; + var locs = doc.Descendants(ns + "loc").Select(e => e.Value).ToList(); + locs.Should().HaveCount(5); + locs.Should().Contain("https://www.elastic.co/docs/api/"); + locs.Should().Contain("https://www.elastic.co/docs/api/doc/elasticsearch/"); + locs.Should().Contain("https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ping"); + locs.Should().Contain("https://www.elastic.co/docs/api/doc/elasticsearch/v8/operation/operation-ping"); + } + [Fact] public void BuildSearchBody_FirstPage_HasPitButNoSearchAfter() { diff --git a/tests/Elastic.Markdown.Tests/Search/OpenApiSearchTitleTests.cs b/tests/Elastic.Markdown.Tests/Search/OpenApiSearchTitleTests.cs index b01e02b0bb..f3d6e5a480 100644 --- a/tests/Elastic.Markdown.Tests/Search/OpenApiSearchTitleTests.cs +++ b/tests/Elastic.Markdown.Tests/Search/OpenApiSearchTitleTests.cs @@ -34,6 +34,98 @@ public void ApiDocs_PreserveTheExporterSSearchTitle() doc.SearchTitle.Should().Contain("_bulk"); } + [Fact] + public void ApiDocs_VersionedPath_RaisesNavigationDepth() + { + var current = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch/operation/operation-ping", + Title = "Ping - Elasticsearch API", + SearchTitle = "Ping - Elasticsearch API - ping" + }; + var versioned = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch/v8/operation/operation-ping", + Title = "Ping - Elasticsearch 8.x API", + SearchTitle = "Ping - Elasticsearch 8.x API - ping" + }; + + ElasticsearchMarkdownExporter.CommonEnrichments(current, null); + ElasticsearchMarkdownExporter.CommonEnrichments(versioned, null); + + current.Navigation.Depth.Should().Be(20); + versioned.Navigation.Depth.Should().Be(40); + versioned.Navigation.Depth.Should().BeGreaterThan(current.Navigation.Depth); + } + + [Fact] + public void ApiDocs_ProductLanding_AssignsNavigationDepth() + { + var current = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch", + Title = "Elasticsearch API", + SearchTitle = "Elasticsearch API" + }; + var versioned = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch/v8", + Title = "Elasticsearch 8.x API", + SearchTitle = "Elasticsearch 8.x API" + }; + + ElasticsearchMarkdownExporter.CommonEnrichments(current, null); + ElasticsearchMarkdownExporter.CommonEnrichments(versioned, null); + + current.Navigation.Depth.Should().Be(10); + versioned.Navigation.Depth.Should().Be(30); + } + + [Fact] + public void ApiDocs_ProductLanding_RanksAboveMatchingOperation() + { + var currentLanding = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch", + Title = "Elasticsearch API", + SearchTitle = "Elasticsearch API" + }; + var currentOperation = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch/operation/operation-ping", + Title = "Ping - Elasticsearch API", + SearchTitle = "Ping - Elasticsearch API - ping" + }; + var versionedLanding = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch/v8", + Title = "Elasticsearch 8.x API", + SearchTitle = "Elasticsearch 8.x API" + }; + var versionedOperation = new DocumentationDocument + { + ContentType = "api", + Path = "/docs/api/doc/elasticsearch/v8/operation/operation-ping", + Title = "Ping - Elasticsearch 8.x API", + SearchTitle = "Ping - Elasticsearch 8.x API - ping" + }; + + ElasticsearchMarkdownExporter.CommonEnrichments(currentLanding, null); + ElasticsearchMarkdownExporter.CommonEnrichments(currentOperation, null); + ElasticsearchMarkdownExporter.CommonEnrichments(versionedLanding, null); + ElasticsearchMarkdownExporter.CommonEnrichments(versionedOperation, null); + + currentLanding.Navigation.Depth.Should().BeLessThan(currentOperation.Navigation.Depth); + versionedLanding.Navigation.Depth.Should().BeLessThan(versionedOperation.Navigation.Depth); + } + [Fact] public void MarkdownDocs_StillGetTheDerivedSearchTitle() {