diff --git a/src/CrestApps.Core.Docs/docs/changelog/1.1.0.md b/src/CrestApps.Core.Docs/docs/changelog/1.1.0.md index 38f47143..73508062 100644 --- a/src/CrestApps.Core.Docs/docs/changelog/1.1.0.md +++ b/src/CrestApps.Core.Docs/docs/changelog/1.1.0.md @@ -19,7 +19,61 @@ page will be updated as changes land after 1.0.0. - upgrades the framework's dependency baseline, including YesSql 6.0 (new `ISession.SaveAsync` signature), the Model Context Protocol 2.0 packages, the GitHub Copilot SDK 1.0.8 (new `PermissionsApi.SetAllowAllAsync` mode-based API), Anthropic 12.39.0, OllamaSharp 5.4.30, the .NET 10.0.10 runtime/extension packages, and the `Microsoft.Extensions.AI` 10.8.3 packages +## Breaking Changes + +- **MCP servers no longer expose tools by default.** In 1.0.0 a server registered with + `WithCrestAppsHandlers()` listed and invoked registered tools by default. In 1.1.0 tool exposure is + opt-in through the new `McpServerOptions` site settings: nothing is listed or callable until you either + add the tool or tool instance name to `McpServerOptions.Tools` or set `McpServerOptions.ExposeAllTools` + to `true`. Hosts upgrading from 1.0.0 that relied on the previous "expose everything" behavior must set + `ExposeAllTools = true` (or populate `Tools`) to keep exposing their tools. Prompts and resources are + unaffected and are still always registered + +## MCP server tool exposure + +- reworks tool exposure into an opt-in allow-list driven by the `McpServerOptions` site settings. Nothing + is exposed by default: an MCP server lists and invokes only the tools and configured + [tool instances](../core/tool-instances.md) named in `McpServerOptions.Tools`, or every tool + and instance when `McpServerOptions.ExposeAllTools` is `true`. The allow-list is enforced by both the + list and call handlers, so a tool that is not exposed can neither be discovered nor invoked. Because + `McpServerOptions` is backed by site settings, operators choose which tools to expose from the admin + settings UI without redeploying. The MVC and Blazor sample hosts add an "Exposed tools" editor to their + MCP server settings + +## Documentation search tool instances + +- adds documentation search [tool instance sources](../core/tool-instances.md), so a host exposes one + callable search function per documentation site it configures. These are ordinary tool instance sources + registered on the tool instances builder with `AddDocumentationSearchSources()` (or the individual + `AddSitemapDocumentationSource()`, `AddSearchIndexDocumentationSource()`, and + `AddAlgoliaDocumentationSource()` methods), so they can be used with or without the MCP server. Each + configured instance binds one site and surfaces as a distinct function the AI model can call, and the + instances are managed and persisted through the existing tool instance store (YesSql or Entity Framework + Core) and UI. The MVC and Blazor sample-host tool instance editors add source-specific field groups so + operators can configure a documentation site (base URL, sitemap or index URL, Algolia + application/index/search-only key, and per-instance result limits) directly from the create and edit + forms +- ships three documentation search strategies, each as its own source and settings model: the sitemap + source crawls a site through its `sitemap.xml` (for example a public Docusaurus site such as + `core.crestapps.com`), the search-index source downloads a prebuilt JSON search index (for example a + MkDocs Material `search_index.json`) and ranks it locally, and the Algolia source forwards queries to + the hosted Algolia DocSearch API. A singleton materializer caches the crawled corpus or downloaded index + per instance and rebuilds it only when the instance changes, so the corpus is reused across calls +- exposes documentation search functions through the MCP server the same way as any other tool instance: + add the instance name to `McpServerOptions.Tools` (or enable `ExposeAllTools`) to make it discoverable + and callable + ## Fixes +- makes the MCP tool allow-list respond to site-settings changes at runtime. The list and call handlers + now read `McpServerOptions` through `IOptionsMonitor` instead of the cached `IOptions`, so exposing or + removing a tool from the admin settings page takes effect without restarting the host +- lets an MCP client invoke a code tool advertised under a function name that differs from its + registration key. The call handler resolves the tool by the published function name (mirroring the list + handler) so a listed tool can always be called +- rethrows cancellation from the documentation search function instead of reporting it as a successful + error result, and returns a stable, non-leaking failure message for other errors +- updates the MVC and Blazor sample hosts to load `@crestapps/bootstrap-select` 1.2.3 for Bootstrap + select styling - fixes post-session processing endlessly retrying and eventually failing when the AI returned a successful (HTTP 200) response that could not be parsed into structured task results. The no-tools structured output path now records a `Failed` result with a diagnostic message instead of silently returning no result, so these responses no longer exhaust all retry attempts. The unparseable-response case is now logged at `Warning` (including a preview of the raw AI response) instead of only at `Debug`, and the recorded task error message now explains that the AI produced no parseable result or there was no content to evaluate. - avoids issuing a post-session AI request when there is no meaningful user content to evaluate. Sessions whose user prompts are empty, whitespace-only, or system-generated are skipped, so a real AI call is only made when there is something to analyze. diff --git a/src/CrestApps.Core.Docs/docs/mcp/server.md b/src/CrestApps.Core.Docs/docs/mcp/server.md index b8ff90cd..24947781 100644 --- a/src/CrestApps.Core.Docs/docs/mcp/server.md +++ b/src/CrestApps.Core.Docs/docs/mcp/server.md @@ -299,12 +299,105 @@ services.Configure(options => ## Tool Exposure -When your application acts as an MCP server, registered AI tools are exposed to external clients. The server endpoint's `WithListToolsHandler` and `WithCallToolHandler` callbacks delegate to your tool registry: +When your application acts as an MCP server, tool exposure is **opt-in**. Nothing is listed or callable by default: the server only exposes the tools and configured [tool instances](../core/tool-instances.md) that you explicitly allow through the `McpServerOptions` site settings. -1. **List tools** — Returns metadata (name, description, JSON schema) for all registered tools -2. **Call tool** — Resolves the tool by name from the registry and invokes it with the provided arguments +```csharp +services.Configure(options => +{ + // Expose specific tools and tool instances by name. + options.Tools = ["crestapps-docs", "weather"]; + + // Or expose every tool and tool instance. + // When set to true, the Tools allow-list above is ignored. + options.ExposeAllTools = true; +}); +``` + +| Property | Effect | +|----------|--------| +| `Tools` | An allow-list of tool and tool instance names to expose. Matching is case-insensitive. | +| `ExposeAllTools` | When `true`, every tool and tool instance is exposed and the allow-list is ignored. | + +Because `McpServerOptions` is backed by site settings, an operator can choose which tools to expose from the admin **Settings → MCP server** page without redeploying. The allow-list is enforced by **both** the list and call handlers, so a tool that is not exposed can neither be discovered nor invoked. + +Both code-registered tools (from `AddCoreAITool()`, see [Custom Tools](../core/tools.md)) and stored tool instances (from any registered [tool instance source](../core/tool-instances.md), such as the [documentation search sources](#exposing-a-documentation-knowledge-base)) participate in the same allow-list. + +## Exposing a documentation knowledge base + +A common reason to run an MCP server is to answer questions from product or framework documentation that lives on a public site (such as a Docusaurus or MkDocs site). Instead of indexing that content into a vector store, the built-in documentation search [tool instance sources](../core/tool-instances.md) let an operator declare a documentation site as a **tool instance** and scan it on demand. + +These sources are ordinary tool instance sources registered on the tool instances builder, so they are usable with or without the MCP server. Each configured instance binds one site and surfaces as its own callable function, so a host can offer "search the CrestApps docs" and "search the Orchard Core docs" as two distinct tools. Instances are persisted and managed through the standard tool instance store and UI, and are exposed to MCP clients through the [allow-list](#tool-exposure) above — nothing is exposed until you opt in. + +### Registering the sources + +```csharp +builder.Services.AddCrestAppsCore(crestApps => crestApps + .AddAISuite(ai => ai + .AddOpenAI() + .AddToolInstances(toolInstances => toolInstances + .AddDocumentationSearchSources() + .AddYesSqlStores() + ) + .AddMcpServer(mcpServer => mcpServer + .AddYesSqlStores() + ) + ) + .AddYesSqlDataStore(configuration => configuration + .UseSqLite("Data Source=app.db;Cache=Shared") + ) +); +``` + +`AddDocumentationSearchSources()` registers all three built-in sources. To register only the ones you need, call the individual `AddSitemapDocumentationSource()`, `AddSearchIndexDocumentationSource()`, and `AddAlgoliaDocumentationSource()` methods instead. Once the sources are registered, operators create configured instances (each bound to one site) through the tool instances UI or store, and each instance becomes a callable function the AI model can invoke. + +### Search strategies + +A documentation site can be indexed in different ways depending on what the generator publishes. Each strategy is its own source with its own settings model, so you pick the one that matches the site. + +| Source | Registration | Best for | How it works | +|--------|--------------|----------|--------------| +| Sitemap crawl | `AddSitemapDocumentationSource()` | Any site that publishes `sitemap.xml` (Docusaurus, MkDocs, and most static sites). | Crawls pages, strips HTML, and ranks locally with keyword scoring. | +| Search index | `AddSearchIndexDocumentationSource()` | MkDocs Material and other sites that publish a fetchable `search_index.json`. | Downloads the prebuilt index once and ranks its entries locally. | +| Algolia DocSearch | `AddAlgoliaDocumentationSource()` | Docusaurus sites (and others) wired to hosted Algolia DocSearch. | Forwards the query to Algolia, which performs the ranking. | + +The registered source names are defined by `DocumentationToolConstants` (`sitemap-documentation`, `search-index-documentation`, `algolia-documentation`), and all three carry the `Knowledgebase` category. + +Each strategy binds a settings model: + +- **`SitemapDocumentationToolSettings`** — `BaseUrl` (site root, for example `https://core.crestapps.com`), optional `SitemapUrl` (defaults to `{BaseUrl}/sitemap.xml`), optional `MaxResults`, and optional `MaxPages`. +- **`SearchIndexDocumentationToolSettings`** — `BaseUrl` (used to resolve relative locations and the default index URL), optional `IndexUrl` (defaults to `{BaseUrl}/search/search_index.json`), and optional `MaxResults`. This targets the MkDocs Material `search_index.json` schema (`{ "docs": [ { "location", "title", "text" } ] }`). +- **`AlgoliaDocumentationToolSettings`** — `ApplicationId`, `ApiKey` (Algolia **search-only** key, never a write key), `IndexName`, and optional `MaxResults`. + +### Example: a public Docusaurus site + +A public Docusaurus site that requires no authentication — such as [core.crestapps.com](https://core.crestapps.com) — only needs the sitemap crawl source. Docusaurus publishes a standard `sitemap.xml` at the site root, so the crawler discovers `{BaseUrl}/sitemap.xml` automatically. + +1. Register the sitemap source (or all sources) as shown above. +2. Create a tool instance from the **Documentation search (sitemap)** source with a **Name** (for example `crestapps-docs`, the name you expose to MCP clients), a clear **Description**, and a **Base URL** of `https://core.crestapps.com`. +3. Expose the instance through the MCP server by adding its name to the allow-list: + +```csharp +services.Configure(options => +{ + options.Tools = ["crestapps-docs"]; +}); +``` + +Because the site is public, no headers, API keys, or credentials are involved — the crawler issues plain anonymous `GET` requests. The first search crawls the site and caches the corpus; later searches reuse the cache. + +### Corpus caching + +The runtime documentation source (the crawled corpus or downloaded index) is built lazily and cached by a singleton `IDocumentationSourceMaterializer`, keyed by the instance identifier. The cache is rebuilt only when the instance changes, so an edit to a site's settings is picked up on the next search while an unchanged instance reuses its corpus across calls. + +### Adding a new documentation source + +To support a documentation site that none of the built-in strategies cover, implement a new [tool instance source](../core/tool-instances.md): + +1. Create a settings model for the user-provided configuration. +2. Implement `IAIToolInstanceSource.CreateTool(AIToolInstance)` to read the settings and return a `DocumentationSearchToolFunction` (or your own `AIFunction`) bound to a concrete `IDocumentationSource`. +3. Register the source on the tool instances builder with `AddSource(name, configure)`. -Tools registered via `AddCoreAITool()` (see [Custom Tools](../core/tools.md)) are automatically available to MCP clients unless they are marked with `.Hidden()`. Hidden tools remain available to explicitly configured profiles and agents, but the shared MCP handlers do not list or invoke them directly. +Operators then create instances of your new source exactly like the built-in ones, and expose them through the same MCP allow-list. ## Server Metadata diff --git a/src/Primitives/CrestApps.Core.AI.Mcp/McpServerBuilderExtensions.cs b/src/Primitives/CrestApps.Core.AI.Mcp/McpServerBuilderExtensions.cs index 010a32e8..2b60a27f 100644 --- a/src/Primitives/CrestApps.Core.AI.Mcp/McpServerBuilderExtensions.cs +++ b/src/Primitives/CrestApps.Core.AI.Mcp/McpServerBuilderExtensions.cs @@ -1,5 +1,6 @@ using CrestApps.Core.AI.Mcp.Services; using CrestApps.Core.AI.Tooling; +using CrestApps.Core.Services; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -7,6 +8,7 @@ using ModelContextProtocol; using ModelContextProtocol.Protocol; using ModelContextProtocol.Server; +using McpServerOptions = CrestApps.Core.AI.Mcp.Models.McpServerOptions; namespace CrestApps.Core.AI.Mcp; @@ -20,6 +22,8 @@ public static class McpServerBuilderExtensions /// This wires the CrestApps tool registry (), /// , and /// into the MCP protocol so both Orchard Core and standalone MVC hosts share the same handler logic. + /// Which tools and tool instances are actually listed and callable is controlled by the + /// site settings allow-list. /// /// The builder. public static IMcpServerBuilder WithCrestAppsHandlers(this IMcpServerBuilder builder) @@ -27,148 +31,300 @@ public static IMcpServerBuilder WithCrestAppsHandlers(this IMcpServerBuilder bui ArgumentNullException.ThrowIfNull(builder); return builder - .WithListToolsHandler((request, cancellationToken) => + .WithListToolsHandler(async (request, cancellationToken) => + { + var serverOptions = request.Services.GetRequiredService>().CurrentValue; + var exposeAll = serverOptions.ExposeAllTools; + var allowList = exposeAll ? null : BuildAllowList(serverOptions.Tools); + var toolDefinitions = request.Services.GetRequiredService>().Value; + ILogger logger = null; + var tools = new List(); + var seenNames = new HashSet(StringComparer.Ordinal); + + foreach (var (name, definition) in toolDefinitions.Tools) + { + if (definition.Hidden || !IsAllowed(exposeAll, allowList, name, definition.Name)) { - var toolDefinitions = request.Services.GetRequiredService>().Value; - ILogger logger = null; - var tools = new List(); + continue; + } - foreach (var (name, _) in toolDefinitions.Tools.Where(tool => !tool.Value.Hidden)) + try + { + if (request.Services.GetKeyedService(name) is AIFunction aiFunction && seenNames.Add(aiFunction.Name)) { - try - { - if (request.Services.GetKeyedService(name) is AIFunction aiFunction) - { - tools.Add(new Tool - { - Name = aiFunction.Name, - Description = aiFunction.Description, - InputSchema = aiFunction.JsonSchema, - }); - } - } - catch (Exception ex) + tools.Add(new Tool { - logger ??= request.Services.GetRequiredService>(); - logger.LogError(ex, "Error creating tool instance for '{ToolName}'.", name); - } + Name = aiFunction.Name, + Description = aiFunction.Description, + InputSchema = aiFunction.JsonSchema, + }); } + } + catch (Exception ex) + { + logger ??= request.Services.GetRequiredService>(); + logger.LogError(ex, "Error creating tool instance for '{ToolName}'.", name); + } + } + + var instanceCatalog = request.Services.GetService>(); - var sdkTools = request.Services.GetService>(); + if (instanceCatalog is not null) + { + var instances = await instanceCatalog.GetAllAsync(cancellationToken); - if (sdkTools is not null) + foreach (var instance in instances) + { + if (string.IsNullOrEmpty(instance.Source)) { - using var sdkToolEnumerator = sdkTools.GetEnumerator(); + continue; + } - if (sdkToolEnumerator.MoveNext()) - { - var toolNames = new HashSet(tools.Count, StringComparer.Ordinal); + var functionName = instance.GetFunctionName(); - foreach (var tool in tools) - { - toolNames.Add(tool.Name); - } + if (!IsAllowed(exposeAll, allowList, functionName, instance.Name)) + { + continue; + } + + var source = request.Services.GetKeyedService(instance.Source); - do + if (source is null) + { + continue; + } + + try + { + if (source.CreateTool(instance) is AIFunction aiFunction && seenNames.Add(aiFunction.Name)) + { + tools.Add(new Tool { - var sdkTool = sdkToolEnumerator.Current; - - if (toolNames.Add(sdkTool.ProtocolTool.Name)) - { - tools.Add(sdkTool.ProtocolTool); - } - } - while (sdkToolEnumerator.MoveNext()); + Name = aiFunction.Name, + Description = aiFunction.Description, + InputSchema = aiFunction.JsonSchema, + }); } } + catch (Exception ex) + { + logger ??= request.Services.GetRequiredService>(); + logger.LogError(ex, "Error creating tool for instance '{InstanceName}'.", instance.Name); + } + } + } + + return new ListToolsResult { Tools = tools }; + }) + .WithCallToolHandler(async (request, cancellationToken) => + { + var serverOptions = request.Services.GetRequiredService>().CurrentValue; + var exposeAll = serverOptions.ExposeAllTools; + var allowList = exposeAll ? null : BuildAllowList(serverOptions.Tools); + var toolDefinitions = request.Services.GetRequiredService>().Value; + + var logger = request.Services.GetService>(); + var codeTool = ResolveAllowedCodeTool(request.Services, toolDefinitions, exposeAll, allowList, request.Params.Name, logger); - return ValueTask.FromResult(new ListToolsResult { Tools = tools }); - }) - .WithCallToolHandler(async (request, cancellationToken) => + if (codeTool is not null) + { + var result = await codeTool.InvokeAsync(BuildArguments(request), cancellationToken); + + return new CallToolResult { - var toolDefinitions = request.Services.GetRequiredService>().Value; + Content = [new TextContentBlock { Text = result?.ToString() ?? string.Empty }], + }; + } - if (toolDefinitions.Tools.TryGetValue(request.Params.Name, out var definition) && - !definition.Hidden) - { - if (request.Services.GetKeyedService(request.Params.Name) is not AIFunction aiFunction) - { - throw new McpException($"Failed to create tool '{request.Params.Name}'."); - } + var instanceCatalog = request.Services.GetService>(); - var arguments = new AIFunctionArguments - { - Services = request.Services, - Context = new Dictionary - { - ["mcpRequest"] = request, - }, - }; + if (instanceCatalog is not null) + { + var instance = await ResolveInstanceAsync(instanceCatalog, request.Params.Name, cancellationToken); - if (request.Params.Arguments is not null) - { - foreach (var kvp in request.Params.Arguments) - { - arguments[kvp.Key] = kvp.Value; - } - } + if (instance is not null && + !string.IsNullOrEmpty(instance.Source) && + IsAllowed(exposeAll, allowList, instance.GetFunctionName(), instance.Name)) + { + var source = request.Services.GetKeyedService(instance.Source); - var result = await aiFunction.InvokeAsync(arguments, cancellationToken); + if (source is not null && source.CreateTool(instance) is AIFunction instanceFunction) + { + var result = await instanceFunction.InvokeAsync(BuildArguments(request), cancellationToken); return new CallToolResult { Content = [new TextContentBlock { Text = result?.ToString() ?? string.Empty }], }; } + } + } - var sdkTools = request.Services.GetService>(); - var sdkTool = sdkTools?.FirstOrDefault(t => t.ProtocolTool.Name == request.Params.Name); + throw new McpException($"Tool '{request.Params.Name}' not found."); + }) + .WithListPromptsHandler(async (request, cancellationToken) => + { + var promptService = request.Services.GetRequiredService(); - if (sdkTool is not null) - { - return await sdkTool.InvokeAsync(request, cancellationToken); - } + return new ListPromptsResult + { + Prompts = await promptService.ListAsync(), + }; + }) + .WithGetPromptHandler(async (request, cancellationToken) => + { + var promptService = request.Services.GetRequiredService(); - throw new McpException($"Tool '{request.Params.Name}' not found."); - }) - .WithListPromptsHandler(async (request, cancellationToken) => - { - var promptService = request.Services.GetRequiredService(); + return await promptService.GetAsync(request, cancellationToken); + }) + .WithListResourcesHandler(async (request, cancellationToken) => + { + var resourceService = request.Services.GetRequiredService(); - return new ListPromptsResult - { - Prompts = await promptService.ListAsync(), - }; - }) - .WithGetPromptHandler(async (request, cancellationToken) => - { - var promptService = request.Services.GetRequiredService(); + return new ListResourcesResult + { + Resources = await resourceService.ListAsync(), + }; + }) + .WithListResourceTemplatesHandler(async (request, cancellationToken) => + { + var resourceService = request.Services.GetRequiredService(); - return await promptService.GetAsync(request, cancellationToken); - }) - .WithListResourcesHandler(async (request, cancellationToken) => - { - var resourceService = request.Services.GetRequiredService(); + return new ListResourceTemplatesResult + { + ResourceTemplates = await resourceService.ListTemplatesAsync(), + }; + }) + .WithReadResourceHandler(async (request, cancellationToken) => + { + var resourceService = request.Services.GetRequiredService(); - return new ListResourcesResult - { - Resources = await resourceService.ListAsync(), - }; - }) - .WithListResourceTemplatesHandler(async (request, cancellationToken) => - { - var resourceService = request.Services.GetRequiredService(); + return await resourceService.ReadAsync(request, cancellationToken); + }); + } - return new ListResourceTemplatesResult - { - ResourceTemplates = await resourceService.ListTemplatesAsync(), - }; - }) - .WithReadResourceHandler(async (request, cancellationToken) => - { - var resourceService = request.Services.GetRequiredService(); + private static HashSet BuildAllowList(IEnumerable names) + { + var allowList = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (names is not null) + { + foreach (var name in names) + { + if (!string.IsNullOrWhiteSpace(name)) + { + allowList.Add(name.Trim()); + } + } + } + + return allowList; + } + + private static bool IsAllowed(bool exposeAll, HashSet allowList, params string[] candidates) + { + if (exposeAll) + { + return true; + } + + foreach (var candidate in candidates) + { + if (!string.IsNullOrEmpty(candidate) && allowList.Contains(candidate)) + { + return true; + } + } + + return false; + } + + private static AIFunction ResolveAllowedCodeTool( + IServiceProvider services, + AIToolDefinitionOptions toolDefinitions, + bool exposeAll, + HashSet allowList, + string protocolName, + ILogger logger) + { + if (toolDefinitions.Tools.TryGetValue(protocolName, out var direct) && + !direct.Hidden && + IsAllowed(exposeAll, allowList, protocolName, direct.Name) && + TryCreateFunction(services, protocolName, logger) is { } directFunction && + string.Equals(directFunction.Name, protocolName, StringComparison.Ordinal)) + { + return directFunction; + } + + foreach (var (name, definition) in toolDefinitions.Tools) + { + if (definition.Hidden || !IsAllowed(exposeAll, allowList, name, definition.Name)) + { + continue; + } + + if (TryCreateFunction(services, name, logger) is { } function && + string.Equals(function.Name, protocolName, StringComparison.Ordinal)) + { + return function; + } + } + + return null; + } + + private static AIFunction TryCreateFunction(IServiceProvider services, string key, ILogger logger) + { + try + { + return services.GetKeyedService(key) as AIFunction; + } + catch (Exception ex) + { + logger?.LogError(ex, "Error creating tool '{ToolName}'.", key); + + return null; + } + } + + private static async Task ResolveInstanceAsync( + INamedCatalog catalog, + string name, + CancellationToken cancellationToken) + { + var instances = await catalog.GetAllAsync(cancellationToken); + + foreach (var instance in instances) + { + if (string.Equals(instance.GetFunctionName(), name, StringComparison.Ordinal) || + (instance.Name is not null && string.Equals(instance.Name, name, StringComparison.OrdinalIgnoreCase))) + { + return instance; + } + } + + return null; + } + + private static AIFunctionArguments BuildArguments(RequestContext request) + { + var arguments = new AIFunctionArguments + { + Services = request.Services, + Context = new Dictionary + { + ["mcpRequest"] = request, + }, + }; + + if (request.Params.Arguments is not null) + { + foreach (var kvp in request.Params.Arguments) + { + arguments[kvp.Key] = kvp.Value; + } + } - return await resourceService.ReadAsync(request, cancellationToken); - }); + return arguments; } } diff --git a/src/Primitives/CrestApps.Core.AI.Mcp/Models/McpServerOptions.cs b/src/Primitives/CrestApps.Core.AI.Mcp/Models/McpServerOptions.cs index 6d6976ef..b9c5a6d0 100644 --- a/src/Primitives/CrestApps.Core.AI.Mcp/Models/McpServerOptions.cs +++ b/src/Primitives/CrestApps.Core.AI.Mcp/Models/McpServerOptions.cs @@ -27,4 +27,20 @@ public sealed class McpServerOptions /// . /// public bool RequireAccessPermission { get; set; } = true; + + /// + /// Gets or sets a value indicating whether every non-hidden tool and configured tool instance is + /// exposed to MCP clients. When false (the default), the server exposes nothing unless a tool + /// or tool instance is explicitly listed in . When true, + /// is ignored and all non-hidden tools and instances are exposed. + /// + public bool ExposeAllTools { get; set; } + + /// + /// Gets or sets the allow-list of tool names exposed to MCP clients when + /// is false. Each entry matches a code-registered tool name, a tool instance's function name, or + /// a tool instance's technical name. The server exposes nothing by default, so a tool or instance is + /// only listed and callable when it appears here. + /// + public IList Tools { get; set; } = []; } diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocSearchSite.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocSearchSite.cs new file mode 100644 index 00000000..a5f0b1c8 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocSearchSite.cs @@ -0,0 +1,35 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Describes a documentation site that is searchable through the Algolia DocSearch query API (the +/// hosted search used by many Docusaurus sites). The built-in source forwards queries to Algolia and +/// maps the returned hits to documentation results without crawling or caching a corpus locally. +/// +public sealed class AlgoliaDocSearchSite +{ + /// + /// Gets or sets the unique logical name of the site. + /// + public string Name { get; set; } + + /// + /// Gets or sets the Algolia application identifier. + /// + public string ApplicationId { get; set; } + + /// + /// Gets or sets the Algolia search-only API key. + /// + public string ApiKey { get; set; } + + /// + /// Gets or sets the Algolia index name to query. + /// + public string IndexName { get; set; } + + /// + /// Gets or sets the maximum number of results this site should contribute to a search. When not + /// set, the global value is used. + /// + public int? MaxResults { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationSource.cs new file mode 100644 index 00000000..5f67f6de --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationSource.cs @@ -0,0 +1,179 @@ +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// A built-in that searches a documentation site through the +/// Algolia DocSearch query API. Queries are forwarded to Algolia, which performs the ranking, and the +/// returned hits are mapped to documentation results. This source does not crawl or cache a corpus +/// locally; each search issues a live query. +/// +public sealed class AlgoliaDocumentationSource : IDocumentationSource +{ + private static readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + private readonly AlgoliaDocSearchSite _site; + private readonly DocumentationSearchOptions _options; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The site configuration. + /// The global documentation search options. + /// The HTTP client factory. + /// The logger. + public AlgoliaDocumentationSource( + AlgoliaDocSearchSite site, + DocumentationSearchOptions options, + IHttpClientFactory httpClientFactory, + ILogger logger) + { + _site = site; + _options = options; + _httpClientFactory = httpClientFactory; + _logger = logger; + } + + /// + public string Name => _site.Name; + + /// + public async Task> SearchAsync(DocumentationSearchRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + if (string.IsNullOrWhiteSpace(request.Query)) + { + return []; + } + + var maxResults = Math.Min(request.MaxResults, _site.MaxResults ?? _options.MaxResultsPerSite); + + if (maxResults <= 0) + { + return []; + } + + try + { + var hits = await QueryAsync(request.Query, maxResults, cancellationToken); + + if (hits.Count == 0) + { + return []; + } + + var results = new List(hits.Count); + + for (var i = 0; i < hits.Count; i++) + { + var hit = hits[i]; + + if (string.IsNullOrWhiteSpace(hit.Url)) + { + continue; + } + + results.Add(new DocumentationSearchResult + { + SourceName = Name, + Title = ResolveTitle(hit), + Url = hit.Url, + Snippet = hit.Content, + Score = hits.Count - i, + }); + } + + return results; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to query Algolia DocSearch for documentation source '{SourceName}'.", _site.Name); + + return []; + } + } + + private async Task> QueryAsync(string query, int maxResults, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(DocumentationToolConstants.HttpClientName); + + var requestUri = $"https://{_site.ApplicationId}-dsn.algolia.net/1/indexes/{Uri.EscapeDataString(_site.IndexName)}/query"; + var parameters = $"query={Uri.EscapeDataString(query)}&hitsPerPage={maxResults}"; + + using var message = new HttpRequestMessage(HttpMethod.Post, requestUri) + { + Content = JsonContent.Create(new AlgoliaQueryRequest { Params = parameters }, options: _serializerOptions), + }; + + message.Headers.TryAddWithoutValidation("X-Algolia-Application-Id", _site.ApplicationId); + message.Headers.TryAddWithoutValidation("X-Algolia-API-Key", _site.ApiKey); + message.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using var response = await client.SendAsync(message, cancellationToken); + + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadFromJsonAsync(_serializerOptions, cancellationToken); + + return payload?.Hits ?? []; + } + + private static string ResolveTitle(AlgoliaHit hit) + { + if (hit.Hierarchy is not null) + { + foreach (var level in new[] { hit.Hierarchy.Lvl6, hit.Hierarchy.Lvl5, hit.Hierarchy.Lvl4, hit.Hierarchy.Lvl3, hit.Hierarchy.Lvl2, hit.Hierarchy.Lvl1, hit.Hierarchy.Lvl0 }) + { + if (!string.IsNullOrWhiteSpace(level)) + { + return level; + } + } + } + + return hit.Url; + } + + private sealed class AlgoliaQueryRequest + { + public string Params { get; set; } + } + + private sealed class AlgoliaQueryResponse + { + public IReadOnlyList Hits { get; set; } + } + + private sealed class AlgoliaHit + { + public string Url { get; set; } + + public string Content { get; set; } + + public AlgoliaHierarchy Hierarchy { get; set; } + } + + private sealed class AlgoliaHierarchy + { + public string Lvl0 { get; set; } + + public string Lvl1 { get; set; } + + public string Lvl2 { get; set; } + + public string Lvl3 { get; set; } + + public string Lvl4 { get; set; } + + public string Lvl5 { get; set; } + + public string Lvl6 { get; set; } + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationToolSettings.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationToolSettings.cs new file mode 100644 index 00000000..a3124734 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationToolSettings.cs @@ -0,0 +1,31 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The user-provided settings for an Algolia DocSearch documentation search tool instance. The settings +/// are persisted in the owning and bind +/// the produced function to a single Algolia index. Only the search-only API key should be supplied here, +/// as it is safe to expose to clients. +/// +public sealed class AlgoliaDocumentationToolSettings +{ + /// + /// Gets or sets the Algolia application identifier. + /// + public string ApplicationId { get; set; } + + /// + /// Gets or sets the Algolia search-only API key. + /// + public string ApiKey { get; set; } + + /// + /// Gets or sets the Algolia index name to query. + /// + public string IndexName { get; set; } + + /// + /// Gets or sets the maximum number of results this instance returns for a single search. When not + /// set, a built-in default is used. + /// + public int? MaxResults { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationToolSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationToolSource.cs new file mode 100644 index 00000000..eabd8263 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/AlgoliaDocumentationToolSource.cs @@ -0,0 +1,54 @@ +using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The built-in that lets users configure documentation search through +/// the hosted Algolia DocSearch query API (the search used by many Docusaurus sites). Each configured +/// binds one Algolia index; the AI model only supplies the search query. +/// +public sealed class AlgoliaDocumentationToolSource : IAIToolInstanceSource +{ + /// + /// Creates the bound to the supplied instance's settings. + /// + /// The configured tool instance whose settings should be bound to the produced tool. + /// The configured documentation search function. + public AITool CreateTool(AIToolInstance instance) + { + ArgumentNullException.ThrowIfNull(instance); + + var settings = instance.GetOrCreate(); + + var functionName = instance.GetFunctionName(); + var description = string.IsNullOrWhiteSpace(instance.Description) + ? "Searches the configured documentation site and returns the most relevant passages with their source URLs." + : instance.Description; + + return new DocumentationSearchToolFunction(functionName, description, instance, services => + { + var site = new AlgoliaDocSearchSite + { + Name = string.IsNullOrWhiteSpace(instance.Name) + ? functionName + : instance.Name, + ApplicationId = settings.ApplicationId, + ApiKey = settings.ApiKey, + IndexName = settings.IndexName, + MaxResults = settings.MaxResults, + }; + + var options = services.GetService>()?.Value ?? new DocumentationSearchOptions(); + var httpClientFactory = services.GetRequiredService(); + var logger = services.GetService()?.CreateLogger() + ?? NullLogger.Instance; + + return new AlgoliaDocumentationSource(site, options, httpClientFactory, logger); + }); + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/CachingDocumentationSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/CachingDocumentationSource.cs new file mode 100644 index 00000000..40d4c35f --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/CachingDocumentationSource.cs @@ -0,0 +1,90 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// A base class for documentation sources that materialize a full +/// and answer queries from it. The corpus is built lazily on first use and cached in memory until it +/// expires, so derived sources only implement how the corpus is produced. +/// +public abstract class CachingDocumentationSource : IDocumentationSource +{ + private readonly TimeSpan _cacheDuration; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _loadLock = new(1, 1); + + private DocumentationCorpus _corpus; + private DateTimeOffset _loadedAt = DateTimeOffset.MinValue; + + /// + /// Initializes a new instance of the class. + /// + /// The unique logical name of the source. + /// How long the built corpus is cached before it is refreshed. + /// The time provider. + protected CachingDocumentationSource(string name, TimeSpan cacheDuration, TimeProvider timeProvider) + { + Name = name; + _cacheDuration = cacheDuration; + _timeProvider = timeProvider; + } + + /// + public string Name { get; } + + /// + /// Gets the maximum number of results this source contributes to a search. + /// + protected abstract int MaxResults { get; } + + /// + /// Builds the documentation corpus that backs this source. + /// + /// The cancellation token. + /// The materialized corpus. + protected abstract Task BuildCorpusAsync(CancellationToken cancellationToken); + + /// + public async Task> SearchAsync(DocumentationSearchRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + if (string.IsNullOrWhiteSpace(request.Query)) + { + return []; + } + + var corpus = await GetCorpusAsync(cancellationToken); + + return corpus.Search(request.Query, Name, Math.Min(request.MaxResults, MaxResults)); + } + + private async Task GetCorpusAsync(CancellationToken cancellationToken) + { + var now = _timeProvider.GetUtcNow(); + + if (_corpus is not null && now - _loadedAt < _cacheDuration) + { + return _corpus; + } + + await _loadLock.WaitAsync(cancellationToken); + + try + { + now = _timeProvider.GetUtcNow(); + + if (_corpus is not null && now - _loadedAt < _cacheDuration) + { + return _corpus; + } + + _corpus = await BuildCorpusAsync(cancellationToken); + _loadedAt = _timeProvider.GetUtcNow(); + + return _corpus; + } + finally + { + _loadLock.Release(); + } + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DefaultDocumentationSourceMaterializer.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DefaultDocumentationSourceMaterializer.cs new file mode 100644 index 00000000..44b32821 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DefaultDocumentationSourceMaterializer.cs @@ -0,0 +1,43 @@ +using System.Collections.Concurrent; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The default . It keeps one cached +/// per key and rebuilds it only when the supplied signature changes, +/// so a crawled corpus or a downloaded search index survives across searches for the lifetime of the +/// application. +/// +public sealed class DefaultDocumentationSourceMaterializer : IDocumentationSourceMaterializer +{ + private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); + + /// + public IDocumentationSource GetOrCreate(string key, string signature, Func factory) + { + ArgumentException.ThrowIfNullOrEmpty(key); + ArgumentNullException.ThrowIfNull(factory); + + var cached = _cache.AddOrUpdate( + key, + _ => new CachedSource(signature, factory()), + (_, existing) => string.Equals(existing.Signature, signature, StringComparison.Ordinal) + ? existing + : new CachedSource(signature, factory())); + + return cached.Source; + } + + private sealed class CachedSource + { + public CachedSource(string signature, IDocumentationSource source) + { + Signature = signature; + Source = source; + } + + public string Signature { get; } + + public IDocumentationSource Source { get; } + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationCorpus.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationCorpus.cs new file mode 100644 index 00000000..f6695df1 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationCorpus.cs @@ -0,0 +1,185 @@ +using System.Text.RegularExpressions; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// An in-memory, keyword-searchable collection of documentation entries. Sources that materialize a +/// full corpus (for example a crawled site or a downloaded search index) build a +/// once and reuse it to answer queries with lightweight keyword +/// scoring, keeping the ranking behavior consistent across sources. +/// +public sealed partial class DocumentationCorpus +{ + private readonly IReadOnlyList _entries; + + /// + /// Initializes a new instance of the class. + /// + /// The documentation entries that make up the corpus. + public DocumentationCorpus(IReadOnlyList entries) + { + _entries = entries ?? []; + } + + /// + /// Gets the number of entries in the corpus. + /// + public int Count => _entries.Count; + + /// + /// Searches the corpus for entries relevant to the supplied query. + /// + /// The free-text query. + /// The logical name of the source that owns this corpus. + /// The maximum number of results to return. + /// The relevant entries ordered by descending relevance. + public IReadOnlyList Search(string query, string sourceName, int maxResults) + { + if (_entries.Count == 0 || string.IsNullOrWhiteSpace(query) || maxResults <= 0) + { + return []; + } + + var terms = Tokenize(query); + + if (terms.Length == 0) + { + return []; + } + + var scored = new List(); + + foreach (var entry in _entries) + { + var score = Score(entry, terms); + + if (score <= 0) + { + continue; + } + + scored.Add(new DocumentationSearchResult + { + SourceName = sourceName, + Title = string.IsNullOrWhiteSpace(entry.Title) ? entry.Url : entry.Title, + Url = entry.Url, + Snippet = BuildSnippet(entry.Text, terms), + Score = score, + }); + } + + return scored + .OrderByDescending(result => result.Score) + .Take(maxResults) + .ToList(); + } + + private static double Score(Entry entry, string[] terms) + { + double score = 0; + var matchedTerms = 0; + + foreach (var term in terms) + { + var bodyCount = CountOccurrences(entry.Text, term); + var titleCount = CountOccurrences(entry.Title, term); + + if (bodyCount == 0 && titleCount == 0) + { + continue; + } + + matchedTerms++; + score += bodyCount + (titleCount * 5); + } + + if (matchedTerms == 0) + { + return 0; + } + + return score * matchedTerms; + } + + private static int CountOccurrences(string text, string term) + { + if (string.IsNullOrEmpty(text)) + { + return 0; + } + + var count = 0; + var index = 0; + + while ((index = text.IndexOf(term, index, StringComparison.OrdinalIgnoreCase)) >= 0) + { + count++; + index += term.Length; + } + + return count; + } + + private static string BuildSnippet(string text, string[] terms) + { + const int windowSize = 240; + + if (string.IsNullOrEmpty(text)) + { + return null; + } + + var matchIndex = -1; + + foreach (var term in terms) + { + var index = text.IndexOf(term, StringComparison.OrdinalIgnoreCase); + + if (index >= 0 && (matchIndex < 0 || index < matchIndex)) + { + matchIndex = index; + } + } + + if (matchIndex < 0) + { + return text.Length <= windowSize ? text : text[..windowSize] + "…"; + } + + var start = Math.Max(0, matchIndex - (windowSize / 2)); + var length = Math.Min(windowSize, text.Length - start); + var snippet = text.Substring(start, length).Trim(); + + if (start > 0) + { + snippet = "…" + snippet; + } + + if (start + length < text.Length) + { + snippet += "…"; + } + + return snippet; + } + + private static string[] Tokenize(string query) + { + return NonWordRegex() + .Split(query.ToLowerInvariant()) + .Where(token => token.Length >= 2) + .Distinct(StringComparer.Ordinal) + .ToArray(); + } + + [GeneratedRegex(@"[^a-z0-9]+")] + private static partial Regex NonWordRegex(); + + /// + /// Represents a single documentation entry that can be searched. + /// + /// The canonical URL of the entry. + /// The title of the entry. + /// The plain-text body of the entry. + public sealed record Entry(string Url, string Title, string Text); +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchIndexSite.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchIndexSite.cs new file mode 100644 index 00000000..47dfdbe9 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchIndexSite.cs @@ -0,0 +1,32 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Describes a documentation site that publishes a prebuilt search index as JSON (for example a MkDocs +/// Material search_index.json). The built-in source downloads the index once, ranks its entries +/// with keyword scoring, and resolves each result URL relative to . +/// +public sealed class DocumentationSearchIndexSite +{ + /// + /// Gets or sets the unique logical name of the site. + /// + public string Name { get; set; } + + /// + /// Gets or sets the base URL of the documentation site. It is used to resolve relative entry + /// locations and, when is not set, to derive the default index URL. + /// + public string BaseUrl { get; set; } + + /// + /// Gets or sets an explicit URL to the search index JSON. When not set, the source resolves it from + /// by appending /search/search_index.json. + /// + public string IndexUrl { get; set; } + + /// + /// Gets or sets the maximum number of results this site should contribute to a search. When not + /// set, the global value is used. + /// + public int? MaxResults { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchOptions.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchOptions.cs new file mode 100644 index 00000000..42d8ed00 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchOptions.cs @@ -0,0 +1,29 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Runtime limits that control how a built-in documentation search source crawls and ranks a single +/// documentation site. A fresh instance with sensible defaults is created for each configured +/// documentation search tool instance. +/// +public sealed class DocumentationSearchOptions +{ + /// + /// Gets or sets the default maximum number of results a single site contributes to a search. + /// + public int MaxResultsPerSite { get; set; } = 5; + + /// + /// Gets or sets the default maximum number of pages the crawler indexes per site. + /// + public int MaxPagesPerSite { get; set; } = 200; + + /// + /// Gets or sets the maximum number of concurrent page requests the crawler issues per site. + /// + public int MaxConcurrentRequests { get; set; } = 4; + + /// + /// Gets or sets how long a crawled site corpus is cached before it is refreshed. + /// + public TimeSpan CacheDuration { get; set; } = TimeSpan.FromHours(1); +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchRequest.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchRequest.cs new file mode 100644 index 00000000..f3f3ec01 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchRequest.cs @@ -0,0 +1,26 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Represents a single documentation search query issued against an . +/// +public sealed class DocumentationSearchRequest +{ + /// + /// Initializes a new instance of the class. + /// + /// The free-text search query. + public DocumentationSearchRequest(string query) + { + Query = query; + } + + /// + /// Gets the free-text search query. + /// + public string Query { get; } + + /// + /// Gets or sets the maximum number of results the source should return. + /// + public int MaxResults { get; set; } = 5; +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchResult.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchResult.cs new file mode 100644 index 00000000..2ce901eb --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchResult.cs @@ -0,0 +1,33 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Represents a single relevant document returned by an . +/// +public sealed class DocumentationSearchResult +{ + /// + /// Gets or sets the logical name of the source that produced this result. + /// + public string SourceName { get; set; } + + /// + /// Gets or sets the title of the matching document. + /// + public string Title { get; set; } + + /// + /// Gets or sets the canonical URL of the matching document. + /// + public string Url { get; set; } + + /// + /// Gets or sets a short text excerpt that highlights the match. + /// + public string Snippet { get; set; } + + /// + /// Gets or sets the relevance score. Higher values indicate a stronger match. Scores are only + /// comparable within a single source. + /// + public double Score { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchToolFunction.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchToolFunction.cs new file mode 100644 index 00000000..97a1f9da --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSearchToolFunction.cs @@ -0,0 +1,205 @@ +using System.Globalization; +using System.Text.Json; +using CrestApps.Core.AI.Extensions; +using CrestApps.Core.AI.Tooling; +using Cysharp.Text; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// An produced by a documentation search tool instance source. Each function is +/// bound to a single configured documentation site and searches only that site, so a host can expose one +/// callable function per documentation source it wants to make available. The runtime +/// is materialized lazily and cached through the +/// so a crawled corpus or downloaded index is reused +/// across calls. +/// +public sealed class DocumentationSearchToolFunction : AIFunction +{ + private static readonly JsonElement _jsonSchema = JsonSerializer.Deserialize( + """ + { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query used to find relevant documentation." + }, + "maxResults": { + "type": "integer", + "description": "Optional. The maximum number of results to return." + } + }, + "required": ["query"], + "additionalProperties": false + } + """); + + private readonly string _name; + private readonly string _description; + private readonly AIToolInstance _instance; + private readonly Func _sourceFactory; + + /// + /// Initializes a new instance of the class. + /// + /// The function name exposed to the AI model. + /// The description exposed to the AI model. + /// The configured tool instance the function belongs to. Used to key the materialized source cache. + /// A factory that builds the runtime documentation source from request services. + public DocumentationSearchToolFunction( + string name, + string description, + AIToolInstance instance, + Func sourceFactory) + { + _name = name; + _description = string.IsNullOrWhiteSpace(description) + ? name + : description; + _instance = instance; + _sourceFactory = sourceFactory; + } + + /// + /// Gets the function name exposed to the AI model. + /// + public override string Name => _name; + + /// + /// Gets the description exposed to the AI model. + /// + public override string Description => _description; + + /// + /// Gets the JSON schema describing the arguments the model may supply. + /// + public override JsonElement JsonSchema => _jsonSchema; + + /// + /// Gets additional metadata applied to the function. + /// + public override IReadOnlyDictionary AdditionalProperties { get; } = new Dictionary + { + ["Strict"] = false, + }; + + /// + /// Searches the configured documentation site for the supplied query. + /// + /// The arguments supplied by the AI model. + /// The cancellation token. + protected override async ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(arguments); + + var services = arguments.Services; + var logger = services?.GetService()?.CreateLogger(); + + if (!arguments.TryGetFirstString("query", out var query) || string.IsNullOrWhiteSpace(query)) + { + logger?.LogWarning("AI tool '{ToolName}' missing required argument 'query'.", _name); + + return "Unable to find a 'query' argument in the arguments parameter."; + } + + if (services is null) + { + return "No services are available to perform the documentation search."; + } + + var request = new DocumentationSearchRequest(query); + + if (arguments.TryGetFirst("maxResults", out var rawMaxResults) && TryConvertToInt32(rawMaxResults, out var maxResults) && maxResults > 0) + { + request.MaxResults = maxResults; + } + + IReadOnlyList results; + + try + { + var materializer = services.GetRequiredService(); + var signature = (_instance.ModifiedUtc ?? _instance.CreatedUtc).Ticks.ToString(CultureInfo.InvariantCulture); + var source = materializer.GetOrCreate(_instance.ItemId, signature, () => _sourceFactory(services)); + + results = await source.SearchAsync(request, cancellationToken) ?? []; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + logger?.LogWarning(ex, "AI tool '{ToolName}' failed to search documentation.", _name); + + return "The documentation search failed. See the server logs for details."; + } + + if (results.Count == 0) + { + return $"No documentation results were found for '{query}'."; + } + + using var builder = ZString.CreateStringBuilder(); + builder.Append("Documentation results for '"); + builder.Append(query); + builder.AppendLine("':"); + + var index = 0; + + foreach (var result in results) + { + index++; + builder.AppendLine(); + builder.Append('['); + builder.Append(index); + builder.Append("] "); + builder.Append(result.Title); + builder.Append(" — "); + builder.Append(result.Url); + builder.AppendLine(); + + if (!string.IsNullOrWhiteSpace(result.Snippet)) + { + builder.AppendLine(result.Snippet); + } + } + + return builder.ToString(); + } + + private static bool TryConvertToInt32(object value, out int result) + { + switch (value) + { + case int intValue: + result = intValue; + + return true; + case long longValue: + result = (int)longValue; + + return true; + case string stringValue when int.TryParse(stringValue, out var parsed): + result = parsed; + + return true; + case JsonElement { ValueKind: JsonValueKind.Number } jsonElement when jsonElement.TryGetInt32(out var jsonInt): + result = jsonInt; + + return true; + case JsonElement { ValueKind: JsonValueKind.String } jsonElement when int.TryParse(jsonElement.GetString(), out var jsonParsed): + result = jsonParsed; + + return true; + default: + result = 0; + + return false; + } + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSite.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSite.cs new file mode 100644 index 00000000..3a4f0cd7 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationSite.cs @@ -0,0 +1,38 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Describes a public documentation site that the built-in documentation search crawler can scan. +/// A site is identified by its name and base URL; the crawler discovers pages through the site's +/// sitemap.xml unless an explicit is supplied. +/// +public sealed class DocumentationSite +{ + /// + /// Gets or sets the unique logical name of the site. The documentation search tool uses this + /// value to let a caller scope a search to a single source. + /// + public string Name { get; set; } + + /// + /// Gets or sets the base URL of the documentation site (for example https://docs.example.com). + /// + public string BaseUrl { get; set; } + + /// + /// Gets or sets an explicit sitemap URL. When not set, the crawler resolves the sitemap from + /// by appending /sitemap.xml. + /// + public string SitemapUrl { get; set; } + + /// + /// Gets or sets the maximum number of results this site should contribute to a search. When not + /// set, the global value is used. + /// + public int? MaxResults { get; set; } + + /// + /// Gets or sets the maximum number of pages the crawler indexes for this site. When not set, the + /// global value is used. + /// + public int? MaxPages { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationToolConstants.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationToolConstants.cs new file mode 100644 index 00000000..ad27dd14 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationToolConstants.cs @@ -0,0 +1,36 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Well-known identifiers for the built-in documentation search tool instance sources. Each source is +/// registered under a unique name that is stored as the +/// of every created from it. +/// +public static class DocumentationToolConstants +{ + /// + /// The registered source name of the sitemap crawling documentation source (for example a public + /// Docusaurus site that exposes a sitemap.xml). + /// + public const string SitemapSourceName = "sitemap-documentation"; + + /// + /// The registered source name of the prebuilt JSON search index documentation source (for example a + /// MkDocs Material search_index.json). + /// + public const string SearchIndexSourceName = "search-index-documentation"; + + /// + /// The registered source name of the hosted Algolia DocSearch documentation source. + /// + public const string AlgoliaSourceName = "algolia-documentation"; + + /// + /// The category applied to the documentation search sources so they are grouped as knowledge-base tools. + /// + public const string Category = "Knowledgebase"; + + /// + /// The name of the named used by the documentation search crawlers. + /// + public const string HttpClientName = "CrestApps.Documentation"; +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationToolInstanceServiceCollectionExtensions.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationToolInstanceServiceCollectionExtensions.cs new file mode 100644 index 00000000..414015c5 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/DocumentationToolInstanceServiceCollectionExtensions.cs @@ -0,0 +1,127 @@ +using CrestApps.Core.Builders; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Localization; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Convenience registration for the built-in documentation search tool instance sources. Each source is a +/// developer-authored blueprint that users configure one or more times as +/// catalog entries, producing one callable documentation search function per configured site. The sources +/// are registered on the tool instances builder so they can be persisted and managed through a store or UI. +/// +public static class DocumentationToolInstanceServiceCollectionExtensions +{ + /// + /// Registers all built-in documentation search sources (sitemap crawling, prebuilt JSON search index, + /// and Algolia DocSearch) on the tool instances builder. + /// + /// The tool instances builder. + /// The tool instances builder, for chaining. + public static CrestAppsAIToolInstancesBuilder AddDocumentationSearchSources(this CrestAppsAIToolInstancesBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder + .AddSitemapDocumentationSource() + .AddSearchIndexDocumentationSource() + .AddAlgoliaDocumentationSource(); + + return builder; + } + + /// + /// Registers the sitemap crawling documentation search source on the tool instances builder so users + /// can create configured instances that search a public site through its sitemap.xml. + /// + /// The tool instances builder. + /// An optional delegate used to override the source display metadata. + /// The tool instances builder, for chaining. + public static CrestAppsAIToolInstancesBuilder AddSitemapDocumentationSource( + this CrestAppsAIToolInstancesBuilder builder, + Action configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + AddSharedServices(builder); + + builder.AddSource(DocumentationToolConstants.SitemapSourceName, entry => + { + entry.DisplayName = new LocalizedString(DocumentationToolConstants.SitemapSourceName, "Documentation search (sitemap)"); + entry.Description = new LocalizedString( + DocumentationToolConstants.SitemapSourceName, + "Searches a documentation site by crawling its sitemap.xml (for example a public Docusaurus site)."); + entry.Category = new LocalizedString(DocumentationToolConstants.Category, DocumentationToolConstants.Category); + + configure?.Invoke(entry); + }); + + return builder; + } + + /// + /// Registers the prebuilt JSON search index documentation search source on the tool instances builder + /// so users can create configured instances that search a site publishing a search_index.json. + /// + /// The tool instances builder. + /// An optional delegate used to override the source display metadata. + /// The tool instances builder, for chaining. + public static CrestAppsAIToolInstancesBuilder AddSearchIndexDocumentationSource( + this CrestAppsAIToolInstancesBuilder builder, + Action configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + AddSharedServices(builder); + + builder.AddSource(DocumentationToolConstants.SearchIndexSourceName, entry => + { + entry.DisplayName = new LocalizedString(DocumentationToolConstants.SearchIndexSourceName, "Documentation search (search index)"); + entry.Description = new LocalizedString( + DocumentationToolConstants.SearchIndexSourceName, + "Searches a documentation site that publishes a prebuilt JSON search index (for example MkDocs Material)."); + entry.Category = new LocalizedString(DocumentationToolConstants.Category, DocumentationToolConstants.Category); + + configure?.Invoke(entry); + }); + + return builder; + } + + /// + /// Registers the Algolia DocSearch documentation search source on the tool instances builder so users + /// can create configured instances that query the hosted Algolia DocSearch API. + /// + /// The tool instances builder. + /// An optional delegate used to override the source display metadata. + /// The tool instances builder, for chaining. + public static CrestAppsAIToolInstancesBuilder AddAlgoliaDocumentationSource( + this CrestAppsAIToolInstancesBuilder builder, + Action configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + AddSharedServices(builder); + + builder.AddSource(DocumentationToolConstants.AlgoliaSourceName, entry => + { + entry.DisplayName = new LocalizedString(DocumentationToolConstants.AlgoliaSourceName, "Documentation search (Algolia DocSearch)"); + entry.Description = new LocalizedString( + DocumentationToolConstants.AlgoliaSourceName, + "Searches a documentation site through the hosted Algolia DocSearch query API."); + entry.Category = new LocalizedString(DocumentationToolConstants.Category, DocumentationToolConstants.Category); + + configure?.Invoke(entry); + }); + + return builder; + } + + private static void AddSharedServices(CrestAppsAIToolInstancesBuilder builder) + { + builder.Services.TryAddSingleton(TimeProvider.System); + builder.Services.TryAddSingleton(); + builder.Services.AddHttpClient(DocumentationToolConstants.HttpClientName); + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/IDocumentationSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/IDocumentationSource.cs new file mode 100644 index 00000000..1fd0286d --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/IDocumentationSource.cs @@ -0,0 +1,23 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Represents a searchable documentation knowledge base. Implement this interface to expose a custom +/// documentation source (for example a search index, an API, or a local corpus) to the documentation +/// search tool. Built-in sources are provided for public sites that expose a sitemap.xml. +/// +public interface IDocumentationSource +{ + /// + /// Gets the unique logical name of this source. Callers can use this value to scope a search to a + /// single source. + /// + string Name { get; } + + /// + /// Searches the source for documents relevant to the supplied request. + /// + /// The search request. + /// The cancellation token. + /// The relevant documents ordered by descending relevance. + Task> SearchAsync(DocumentationSearchRequest request, CancellationToken cancellationToken); +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/IDocumentationSourceMaterializer.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/IDocumentationSourceMaterializer.cs new file mode 100644 index 00000000..ac79f419 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/IDocumentationSourceMaterializer.cs @@ -0,0 +1,19 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// Caches the runtime materialized for each configured documentation +/// search tool instance so an expensive crawled corpus or downloaded index is reused across searches. +/// A cached source is rebuilt only when the instance that defines it changes. +/// +public interface IDocumentationSourceMaterializer +{ + /// + /// Gets the cached documentation source for the supplied key, creating it with + /// when it is missing or when the cached no longer matches. + /// + /// A stable key that identifies the defining instance (for example its item id). + /// A value that changes whenever the instance's settings change. + /// The factory used to build the source when it must be created or rebuilt. + /// The cached or newly created documentation source. + IDocumentationSource GetOrCreate(string key, string signature, Func factory); +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationSource.cs new file mode 100644 index 00000000..61c7177a --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationSource.cs @@ -0,0 +1,118 @@ +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// A built-in that indexes a documentation site by downloading a +/// prebuilt search index published as JSON (for example a MkDocs Material search_index.json), +/// then ranks its entries against a query using lightweight keyword scoring. The downloaded corpus is +/// cached in memory and refreshed based on . +/// +public sealed class SearchIndexDocumentationSource : CachingDocumentationSource +{ + private static readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + private readonly DocumentationSearchIndexSite _site; + private readonly DocumentationSearchOptions _options; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The site configuration. + /// The global documentation search options. + /// The HTTP client factory. + /// The time provider. + /// The logger. + public SearchIndexDocumentationSource( + DocumentationSearchIndexSite site, + DocumentationSearchOptions options, + IHttpClientFactory httpClientFactory, + TimeProvider timeProvider, + ILogger logger) + : base(site.Name, options.CacheDuration, timeProvider) + { + _site = site; + _options = options; + _httpClientFactory = httpClientFactory; + _logger = logger; + } + + /// + protected override int MaxResults => _site.MaxResults ?? _options.MaxResultsPerSite; + + /// + protected override async Task BuildCorpusAsync(CancellationToken cancellationToken) + { + var indexUrl = ResolveIndexUrl(); + + try + { + var client = _httpClientFactory.CreateClient(DocumentationToolConstants.HttpClientName); + var index = await client.GetFromJsonAsync(indexUrl, _serializerOptions, cancellationToken); + + if (index?.Docs is null || index.Docs.Count == 0) + { + return new DocumentationCorpus([]); + } + + var entries = new List(index.Docs.Count); + + foreach (var doc in index.Docs) + { + if (string.IsNullOrWhiteSpace(doc.Location) || string.IsNullOrWhiteSpace(doc.Text)) + { + continue; + } + + entries.Add(new DocumentationCorpus.Entry(ResolveUrl(doc.Location), doc.Title, doc.Text)); + } + + return new DocumentationCorpus(entries); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read search index '{IndexUrl}' for documentation source '{SourceName}'.", indexUrl, _site.Name); + + return new DocumentationCorpus([]); + } + } + + private string ResolveIndexUrl() + { + if (!string.IsNullOrWhiteSpace(_site.IndexUrl)) + { + return _site.IndexUrl; + } + + return $"{_site.BaseUrl.TrimEnd('/')}/search/search_index.json"; + } + + private string ResolveUrl(string location) + { + if (Uri.TryCreate(location, UriKind.Absolute, out _)) + { + return location; + } + + return $"{_site.BaseUrl.TrimEnd('/')}/{location.TrimStart('/')}"; + } + + private sealed class SearchIndexDocument + { + public IReadOnlyList Docs { get; set; } + } + + private sealed class SearchIndexEntry + { + public string Location { get; set; } + + public string Title { get; set; } + + public string Text { get; set; } + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationToolSettings.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationToolSettings.cs new file mode 100644 index 00000000..bcd99f4e --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationToolSettings.cs @@ -0,0 +1,28 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The user-provided settings for a prebuilt search index documentation search tool instance. The +/// settings are persisted in the owning +/// and bind the produced function to a single documentation site that publishes a JSON search index (for +/// example a MkDocs Material search_index.json). +/// +public sealed class SearchIndexDocumentationToolSettings +{ + /// + /// Gets or sets the base URL of the documentation site. It is used to resolve relative entry + /// locations and, when is not set, to derive the default index URL. + /// + public string BaseUrl { get; set; } + + /// + /// Gets or sets an explicit URL to the search index JSON. When not set, the source resolves it from + /// by appending /search/search_index.json. + /// + public string IndexUrl { get; set; } + + /// + /// Gets or sets the maximum number of results this instance returns for a single search. When not + /// set, a built-in default is used. + /// + public int? MaxResults { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationToolSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationToolSource.cs new file mode 100644 index 00000000..1e116bb2 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SearchIndexDocumentationToolSource.cs @@ -0,0 +1,55 @@ +using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The built-in that lets users configure documentation search over a +/// single site that publishes a prebuilt JSON search index (for example a MkDocs Material +/// search_index.json). Each configured binds one site; the AI model +/// only supplies the search query. +/// +public sealed class SearchIndexDocumentationToolSource : IAIToolInstanceSource +{ + /// + /// Creates the bound to the supplied instance's settings. + /// + /// The configured tool instance whose settings should be bound to the produced tool. + /// The configured documentation search function. + public AITool CreateTool(AIToolInstance instance) + { + ArgumentNullException.ThrowIfNull(instance); + + var settings = instance.GetOrCreate(); + + var functionName = instance.GetFunctionName(); + var description = string.IsNullOrWhiteSpace(instance.Description) + ? "Searches the configured documentation site and returns the most relevant passages with their source URLs." + : instance.Description; + + return new DocumentationSearchToolFunction(functionName, description, instance, services => + { + var site = new DocumentationSearchIndexSite + { + Name = string.IsNullOrWhiteSpace(instance.Name) + ? functionName + : instance.Name, + BaseUrl = settings.BaseUrl, + IndexUrl = settings.IndexUrl, + MaxResults = settings.MaxResults, + }; + + var options = services.GetService>()?.Value ?? new DocumentationSearchOptions(); + var httpClientFactory = services.GetRequiredService(); + var timeProvider = services.GetService() ?? TimeProvider.System; + var logger = services.GetService()?.CreateLogger() + ?? NullLogger.Instance; + + return new SearchIndexDocumentationSource(site, options, httpClientFactory, timeProvider, logger); + }); + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationSource.cs new file mode 100644 index 00000000..f4070f70 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationSource.cs @@ -0,0 +1,197 @@ +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// A built-in that indexes a public documentation site by reading +/// its sitemap.xml, fetching each page, and ranking pages against a query using lightweight +/// keyword scoring. The crawled corpus is cached in memory and refreshed based on +/// . +/// +public sealed partial class SitemapDocumentationSource : CachingDocumentationSource +{ + private readonly DocumentationSite _site; + private readonly DocumentationSearchOptions _options; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The site configuration. + /// The global documentation search options. + /// The HTTP client factory. + /// The time provider. + /// The logger. + public SitemapDocumentationSource( + DocumentationSite site, + DocumentationSearchOptions options, + IHttpClientFactory httpClientFactory, + TimeProvider timeProvider, + ILogger logger) + : base(site.Name, options.CacheDuration, timeProvider) + { + _site = site; + _options = options; + _httpClientFactory = httpClientFactory; + _logger = logger; + } + + /// + protected override int MaxResults => _site.MaxResults ?? _options.MaxResultsPerSite; + + /// + protected override async Task BuildCorpusAsync(CancellationToken cancellationToken) + { + var entries = await CrawlAsync(cancellationToken); + + return new DocumentationCorpus(entries); + } + + private async Task> CrawlAsync(CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(DocumentationToolConstants.HttpClientName); + var urls = await GetSitemapUrlsAsync(client, cancellationToken); + + if (urls.Count == 0) + { + return []; + } + + var maxPages = _site.MaxPages ?? _options.MaxPagesPerSite; + + if (urls.Count > maxPages) + { + urls = urls.Take(maxPages).ToList(); + } + + var pages = new List(urls.Count); + using var throttle = new SemaphoreSlim(Math.Max(1, _options.MaxConcurrentRequests)); + + var tasks = urls.Select(async url => + { + await throttle.WaitAsync(cancellationToken); + + try + { + return await FetchPageAsync(client, url, cancellationToken); + } + finally + { + throttle.Release(); + } + }); + + foreach (var page in await Task.WhenAll(tasks)) + { + if (page is not null) + { + pages.Add(page); + } + } + + return pages; + } + + private async Task> GetSitemapUrlsAsync(HttpClient client, CancellationToken cancellationToken) + { + var sitemapUrl = ResolveSitemapUrl(); + + try + { + var xml = await client.GetStringAsync(sitemapUrl, cancellationToken); + var document = XDocument.Parse(xml); + + var urls = document.Descendants() + .Where(element => string.Equals(element.Name.LocalName, "loc", StringComparison.OrdinalIgnoreCase)) + .Select(element => element.Value?.Trim()) + .Where(value => !string.IsNullOrEmpty(value) && !value.EndsWith(".xml", StringComparison.OrdinalIgnoreCase)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + return urls; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read sitemap '{SitemapUrl}' for documentation source '{SourceName}'.", sitemapUrl, _site.Name); + + return []; + } + } + + private async Task FetchPageAsync(HttpClient client, string url, CancellationToken cancellationToken) + { + try + { + var html = await client.GetStringAsync(url, cancellationToken); + var title = ExtractTitle(html); + var text = ExtractText(html); + + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + return new DocumentationCorpus.Entry(url, title, text); + } + catch (Exception ex) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug(ex, "Failed to fetch documentation page '{Url}' for source '{SourceName}'.", url, _site.Name); + } + + return null; + } + } + + private string ResolveSitemapUrl() + { + if (!string.IsNullOrWhiteSpace(_site.SitemapUrl)) + { + return _site.SitemapUrl; + } + + return $"{_site.BaseUrl.TrimEnd('/')}/sitemap.xml"; + } + + private static string ExtractTitle(string html) + { + var match = TitleRegex().Match(html); + + if (!match.Success) + { + return null; + } + + return System.Net.WebUtility.HtmlDecode(match.Groups[1].Value).Trim(); + } + + private static string ExtractText(string html) + { + var withoutScripts = ScriptRegex().Replace(html, " "); + var withoutStyles = StyleRegex().Replace(withoutScripts, " "); + var withoutTags = TagRegex().Replace(withoutStyles, " "); + var decoded = System.Net.WebUtility.HtmlDecode(withoutTags); + + return WhitespaceRegex().Replace(decoded, " ").Trim(); + } + + [GeneratedRegex(@"]*>.*?", RegexOptions.IgnoreCase | RegexOptions.Singleline)] + private static partial Regex ScriptRegex(); + + [GeneratedRegex(@"]*>.*?", RegexOptions.IgnoreCase | RegexOptions.Singleline)] + private static partial Regex StyleRegex(); + + [GeneratedRegex(@"]*>(.*?)", RegexOptions.IgnoreCase | RegexOptions.Singleline)] + private static partial Regex TitleRegex(); + + [GeneratedRegex("<[^>]+>")] + private static partial Regex TagRegex(); + + [GeneratedRegex(@"\s+")] + private static partial Regex WhitespaceRegex(); +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationToolSettings.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationToolSettings.cs new file mode 100644 index 00000000..2cebc196 --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationToolSettings.cs @@ -0,0 +1,33 @@ +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The user-provided settings for a sitemap crawling documentation search tool instance. The settings +/// are persisted in the owning and bind +/// the produced function to a single documentation site whose pages are discovered through its +/// sitemap.xml. +/// +public sealed class SitemapDocumentationToolSettings +{ + /// + /// Gets or sets the base URL of the documentation site (for example https://core.crestapps.com). + /// + public string BaseUrl { get; set; } + + /// + /// Gets or sets an explicit sitemap URL. When not set, the crawler resolves the sitemap from + /// by appending /sitemap.xml. + /// + public string SitemapUrl { get; set; } + + /// + /// Gets or sets the maximum number of results this instance returns for a single search. When not + /// set, a built-in default is used. + /// + public int? MaxResults { get; set; } + + /// + /// Gets or sets the maximum number of pages the crawler indexes for this site. When not set, a + /// built-in default is used. + /// + public int? MaxPages { get; set; } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationToolSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationToolSource.cs new file mode 100644 index 00000000..8094159a --- /dev/null +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/Documentation/SitemapDocumentationToolSource.cs @@ -0,0 +1,55 @@ +using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace CrestApps.Core.AI.Tooling.Instances.Documentation; + +/// +/// The built-in that lets users configure documentation search over a +/// single site crawled through its sitemap.xml (for example a public Docusaurus site). Each +/// configured binds one site; the AI model only supplies the search query. +/// +public sealed class SitemapDocumentationToolSource : IAIToolInstanceSource +{ + /// + /// Creates the bound to the supplied instance's settings. + /// + /// The configured tool instance whose settings should be bound to the produced tool. + /// The configured documentation search function. + public AITool CreateTool(AIToolInstance instance) + { + ArgumentNullException.ThrowIfNull(instance); + + var settings = instance.GetOrCreate(); + + var functionName = instance.GetFunctionName(); + var description = string.IsNullOrWhiteSpace(instance.Description) + ? "Searches the configured documentation site and returns the most relevant passages with their source URLs." + : instance.Description; + + return new DocumentationSearchToolFunction(functionName, description, instance, services => + { + var site = new DocumentationSite + { + Name = string.IsNullOrWhiteSpace(instance.Name) + ? functionName + : instance.Name, + BaseUrl = settings.BaseUrl, + SitemapUrl = settings.SitemapUrl, + MaxResults = settings.MaxResults, + MaxPages = settings.MaxPages, + }; + + var options = services.GetService>()?.Value ?? new DocumentationSearchOptions(); + var httpClientFactory = services.GetRequiredService(); + var timeProvider = services.GetService() ?? TimeProvider.System; + var logger = services.GetService()?.CreateLogger() + ?? NullLogger.Instance; + + return new SitemapDocumentationSource(site, options, httpClientFactory, timeProvider, logger); + }); + } +} diff --git a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/HttpApiRequestToolInstanceSource.cs b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/HttpApiRequestToolInstanceSource.cs index 33c53db7..d6aac268 100644 --- a/src/Primitives/CrestApps.Core.AI/Tooling/Instances/HttpApiRequestToolInstanceSource.cs +++ b/src/Primitives/CrestApps.Core.AI/Tooling/Instances/HttpApiRequestToolInstanceSource.cs @@ -21,9 +21,7 @@ public AITool CreateTool(AIToolInstance instance) { ArgumentNullException.ThrowIfNull(instance); - var settings = instance.TryGet(out var stored) - ? stored - : new HttpApiRequestToolSettings(); + var settings = instance.GetOrCreate(); var functionName = instance.GetFunctionName(); var description = string.IsNullOrWhiteSpace(instance.Description) diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/App.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/App.razor index 8ed49f4c..60d8fa99 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/App.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/App.razor @@ -6,6 +6,7 @@ @(PageTitle ?? "Blazor AI Integration Sample") + @@ -15,6 +16,7 @@ + diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor index 76a0970c..1ff87b23 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Admin/Settings/Index.razor @@ -12,13 +12,16 @@ @using CrestApps.Core.AI.Models @using CrestApps.Core.AI.Profiles @using CrestApps.Core.AI.Security +@using CrestApps.Core.AI.Tooling @using CrestApps.Core.Blazor.Web.Areas.AIChat.Services @using CrestApps.Core.Blazor.Web.Areas.ChatInteractions.Models @using CrestApps.Core.Blazor.Web.Models @using CrestApps.Core.Blazor.Web.Services @using CrestApps.Core.Blazor.Web.ViewModels @using CrestApps.Core.Infrastructure.Indexing +@using CrestApps.Core.Services @using Microsoft.AspNetCore.DataProtection +@using Microsoft.Extensions.Options @inject SiteSettingsStore SiteSettings @inject IAIDeploymentManager DeploymentManager @@ -28,6 +31,8 @@ @inject ClaudeClientService ClaudeClientService @inject SiteSettingsChangedNotifier SettingsChangedNotifier @inject ToastNotificationService ToastNotifications +@inject IOptions ToolOptionsAccessor +@inject ISourceCatalog ToolInstanceCatalog AI Settings - Blazor AI Integration Sample @@ -724,6 +729,79 @@ else + +
+

+ Choose which tools and configured tool instances are exposed to MCP clients. Nothing is + exposed by default; select the tools to allow, or expose everything. +

+
+ + +
+ @if (!_model.McpServerExposeAllTools) + { +
+
Exposed tools
+ @if (_model.McpServerAvailableTools.Count > 0 || _model.McpServerAvailableToolInstances.Count > 0) + { +
+

Select the tools and configured tool instances MCP clients can discover and invoke.

+
+ + +
+
+ } + @if (_model.McpServerAvailableTools.Count == 0) + { +
No AI tools are registered.
+ } + else + { + @foreach (var group in _model.McpServerAvailableTools.GroupBy(tool => tool.Category).OrderBy(group => group.Key)) + { +
@group.Key
+ @foreach (var tool in group) + { +
+ + +
+ } + } + } + +
Exposed tool instances
+ @if (_model.McpServerAvailableToolInstances.Count == 0) + { +
No tool instances are configured. Add them under AI Tool Instances first.
+ } + else + { + @foreach (var instance in _model.McpServerAvailableToolInstances) + { +
+ + +
+ } + } +
+ } @@ -870,6 +948,8 @@ else McpServerAuthenticationType = mcpServerSettings.AuthenticationType, McpServerApiKey = mcpServerSettings.ApiKey, McpServerRequireAccessPermission = mcpServerSettings.RequireAccessPermission, + McpServerExposeAllTools = mcpServerSettings.ExposeAllTools, + McpServerSelectedToolNames = mcpServerSettings.Tools?.ToArray() ?? [], CopilotAuthenticationType = copilotSettings.AuthenticationType, CopilotClientId = copilotSettings.ClientId, @@ -941,9 +1021,42 @@ else .Select(p => new KeyValuePair(p.ItemId, p.DisplayText ?? p.Name)) .ToList(); + await PopulateMcpServerToolsAsync(); await PopulateClaudeModelsAsync(); } + private async Task PopulateMcpServerToolsAsync() + { + var selectedNames = new HashSet(_model.McpServerSelectedToolNames ?? [], StringComparer.OrdinalIgnoreCase); + + _model.McpServerAvailableTools = ToolOptionsAccessor.Value.Tools + .Where(tool => !tool.Value.Hidden) + .Select(tool => new McpServerToolSelectionItem + { + Name = tool.Key, + Title = tool.Value.Title ?? tool.Key, + Description = tool.Value.Description, + Category = tool.Value.Category ?? "Miscellaneous", + IsSelected = selectedNames.Contains(tool.Key) || selectedNames.Contains(tool.Value.Name), + }) + .OrderBy(tool => tool.Category, StringComparer.OrdinalIgnoreCase) + .ThenBy(tool => tool.Title, StringComparer.OrdinalIgnoreCase) + .ToList(); + + _model.McpServerAvailableToolInstances = (await ToolInstanceCatalog.GetAllAsync()) + .Where(instance => !string.IsNullOrWhiteSpace(instance.Name)) + .OrderBy(instance => instance.Name, StringComparer.OrdinalIgnoreCase) + .Select(instance => new McpServerToolInstanceSelectionItem + { + ItemId = instance.ItemId, + Name = instance.Name, + Description = instance.Description, + Source = instance.Source, + IsSelected = selectedNames.Contains(instance.Name), + }) + .ToList(); + } + private void OnCopilotAuthTypeChanged(ChangeEventArgs e) { if (Enum.TryParse(e.Value?.ToString(), out var value)) @@ -965,6 +1078,65 @@ else } } + private void ToggleMcpServerTool(string name, bool value) + { + var tool = _model.McpServerAvailableTools.FirstOrDefault(item => item.Name == name); + + if (tool is not null) + { + tool.IsSelected = value; + } + } + + private void ToggleMcpServerToolInstance(string itemId, bool value) + { + var instance = _model.McpServerAvailableToolInstances.FirstOrDefault(item => item.ItemId == itemId); + + if (instance is not null) + { + instance.IsSelected = value; + } + } + + private void SelectAllMcpServerTools() + { + foreach (var tool in _model.McpServerAvailableTools) + { + tool.IsSelected = true; + } + + foreach (var instance in _model.McpServerAvailableToolInstances) + { + instance.IsSelected = true; + } + } + + private void DeselectAllMcpServerTools() + { + foreach (var tool in _model.McpServerAvailableTools) + { + tool.IsSelected = false; + } + + foreach (var instance in _model.McpServerAvailableToolInstances) + { + instance.IsSelected = false; + } + } + + private string[] GetSelectedMcpServerToolNames() + { + return _model.McpServerAvailableTools + .Where(tool => tool.IsSelected) + .Select(tool => tool.Name) + .Concat(_model.McpServerAvailableToolInstances + .Where(instance => instance.IsSelected) + .Select(instance => instance.Name)) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + private void OnAnthropicAuthTypeChanged(ChangeEventArgs e) { if (Enum.TryParse(e.Value?.ToString(), out var value)) @@ -1011,6 +1183,7 @@ else private async Task HandleSaveAsync() { _errors.Clear(); + _model.McpServerSelectedToolNames = GetSelectedMcpServerToolNames(); if (_model.MaximumIterationsPerRequest < 1) { @@ -1182,6 +1355,10 @@ else AuthenticationType = _model.McpServerAuthenticationType, ApiKey = _model.McpServerApiKey?.Trim(), RequireAccessPermission = _model.McpServerRequireAccessPermission, + ExposeAllTools = _model.McpServerExposeAllTools, + Tools = _model.McpServerExposeAllTools + ? [] + : _model.McpServerSelectedToolNames.ToList(), }); SiteSettings.Set(new MemoryMetadata diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Create.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Create.razor index bb47e254..645ee549 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Create.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Create.razor @@ -5,6 +5,7 @@ @using CrestApps.Core.AI @using CrestApps.Core.AI.Tooling @using CrestApps.Core.AI.Tooling.Instances +@using CrestApps.Core.AI.Tooling.Instances.Documentation @using CrestApps.Core.Blazor.Web.ViewModels @using CrestApps.Core.Services @using Microsoft.Extensions.Options @@ -203,6 +204,87 @@ } + @if (string.Equals(_model.Source, DocumentationToolConstants.SitemapSourceName, StringComparison.OrdinalIgnoreCase)) + { +
Documentation site (sitemap crawl)
+ +
+ + +
The root URL of the documentation site to crawl.
+ +
+ +
+ + +
Optional. When empty, /sitemap.xml is appended to the base URL.
+
+ +
+
+ + +
+
+ + +
+
+ } + + @if (string.Equals(_model.Source, DocumentationToolConstants.SearchIndexSourceName, StringComparison.OrdinalIgnoreCase)) + { +
Documentation site (prebuilt search index)
+ +
+ + +
The root URL of the documentation site. Used to resolve relative entry locations.
+ +
+ +
+ + +
Optional. When empty, /search/search_index.json is appended to the base URL.
+
+ +
+ + +
+ } + + @if (string.Equals(_model.Source, DocumentationToolConstants.AlgoliaSourceName, StringComparison.OrdinalIgnoreCase)) + { +
Documentation site (Algolia DocSearch)
+ +
+ + + +
+ +
+ + +
Use the public, search-only key. It is safe to expose to clients and is stored without additional protection.
+ +
+ +
+ + + +
+ +
+ + +
+ } +
Cancel @@ -272,6 +354,42 @@ _errors.Add("A description is required so the AI model can tell instances apart."); } + if (string.Equals(model.Source, DocumentationToolConstants.SitemapSourceName, StringComparison.OrdinalIgnoreCase)) + { + ValidateAbsoluteUrl(model.SitemapBaseUrl, "Base URL", required: true); + ValidateAbsoluteUrl(model.SitemapUrl, "Sitemap URL", required: false); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.SearchIndexSourceName, StringComparison.OrdinalIgnoreCase)) + { + ValidateAbsoluteUrl(model.SearchIndexBaseUrl, "Base URL", required: true); + ValidateAbsoluteUrl(model.SearchIndexUrl, "Search index URL", required: false); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.AlgoliaSourceName, StringComparison.OrdinalIgnoreCase)) + { + if (string.IsNullOrWhiteSpace(model.AlgoliaApplicationId)) + { + _errors.Add("Application ID is required."); + } + + if (string.IsNullOrWhiteSpace(model.AlgoliaApiKey)) + { + _errors.Add("Search-only API key is required."); + } + + if (string.IsNullOrWhiteSpace(model.AlgoliaIndexName)) + { + _errors.Add("Index name is required."); + } + + return; + } + if (!string.Equals(model.Source, HttpApiRequestToolConstants.SourceName, StringComparison.OrdinalIgnoreCase)) { return; @@ -381,12 +499,68 @@ string.Equals(entry.Name, name, StringComparison.OrdinalIgnoreCase)); } + private void ValidateAbsoluteUrl(string value, string label, bool required) + { + if (string.IsNullOrWhiteSpace(value)) + { + if (required) + { + _errors.Add($"{label} is required."); + } + + return; + } + + if (!Uri.TryCreate(value, UriKind.Absolute, out _)) + { + _errors.Add($"{label} must be a valid absolute URL."); + } + } + private void Apply(AIToolInstanceViewModel model, AIToolInstance instance) { instance.Name = model.Name.Trim(); instance.Description = model.Description.Trim(); instance.ModifiedUtc = TimeProvider.GetUtcNow().UtcDateTime; + if (string.Equals(model.Source, DocumentationToolConstants.SitemapSourceName, StringComparison.OrdinalIgnoreCase)) + { + instance.Put(new SitemapDocumentationToolSettings + { + BaseUrl = model.SitemapBaseUrl?.Trim(), + SitemapUrl = string.IsNullOrWhiteSpace(model.SitemapUrl) ? null : model.SitemapUrl.Trim(), + MaxResults = model.SitemapMaxResults, + MaxPages = model.SitemapMaxPages, + }); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.SearchIndexSourceName, StringComparison.OrdinalIgnoreCase)) + { + instance.Put(new SearchIndexDocumentationToolSettings + { + BaseUrl = model.SearchIndexBaseUrl?.Trim(), + IndexUrl = string.IsNullOrWhiteSpace(model.SearchIndexUrl) ? null : model.SearchIndexUrl.Trim(), + MaxResults = model.SearchIndexMaxResults, + }); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.AlgoliaSourceName, StringComparison.OrdinalIgnoreCase)) + { + instance.Put(new AlgoliaDocumentationToolSettings + { + ApplicationId = model.AlgoliaApplicationId?.Trim(), + ApiKey = model.AlgoliaApiKey?.Trim(), + IndexName = model.AlgoliaIndexName?.Trim(), + MaxResults = model.AlgoliaMaxResults, + }); + + return; + } + var protector = DataProtectionProvider.CreateProtector(HttpApiRequestToolConstants.DataProtectionPurpose); var existing = instance.TryGet(out var stored) ? stored : new HttpApiRequestToolSettings(); diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Edit.razor b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Edit.razor index df04ccf1..eef44c7d 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Edit.razor +++ b/src/Startup/CrestApps.Core.Blazor.Web/Components/Pages/Tooling/ToolInstances/Edit.razor @@ -4,6 +4,7 @@ @using CrestApps.Core.AI @using CrestApps.Core.AI.Tooling @using CrestApps.Core.AI.Tooling.Instances +@using CrestApps.Core.AI.Tooling.Instances.Documentation @using CrestApps.Core.Blazor.Web.ViewModels @using CrestApps.Core.Services @using Microsoft.Extensions.Options @@ -11,6 +12,7 @@ @inject IDataProtectionProvider DataProtectionProvider @inject IStoreCommitter StoreCommitter @inject NavigationManager Navigation +@inject TimeProvider TimeProvider @inject IOptions AIOptions Edit Tool Instance @@ -224,6 +226,87 @@ else
} + @if (string.Equals(_model.Source, DocumentationToolConstants.SitemapSourceName, StringComparison.OrdinalIgnoreCase)) + { +
Documentation site (sitemap crawl)
+ +
+ + +
The root URL of the documentation site to crawl.
+ +
+ +
+ + +
Optional. When empty, /sitemap.xml is appended to the base URL.
+
+ +
+
+ + +
+
+ + +
+
+ } + + @if (string.Equals(_model.Source, DocumentationToolConstants.SearchIndexSourceName, StringComparison.OrdinalIgnoreCase)) + { +
Documentation site (prebuilt search index)
+ +
+ + +
The root URL of the documentation site. Used to resolve relative entry locations.
+ +
+ +
+ + +
Optional. When empty, /search/search_index.json is appended to the base URL.
+
+ +
+ + +
+ } + + @if (string.Equals(_model.Source, DocumentationToolConstants.AlgoliaSourceName, StringComparison.OrdinalIgnoreCase)) + { +
Documentation site (Algolia DocSearch)
+ +
+ + + +
+ +
+ + +
Use the public, search-only key. It is safe to expose to clients and is stored without additional protection.
+ +
+ +
+ + + +
+ +
+ + +
+ } +
Cancel @@ -304,6 +387,42 @@ else _errors.Add("A description is required so the AI model can tell instances apart."); } + if (string.Equals(model.Source, DocumentationToolConstants.SitemapSourceName, StringComparison.OrdinalIgnoreCase)) + { + ValidateAbsoluteUrl(model.SitemapBaseUrl, "Base URL", required: true); + ValidateAbsoluteUrl(model.SitemapUrl, "Sitemap URL", required: false); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.SearchIndexSourceName, StringComparison.OrdinalIgnoreCase)) + { + ValidateAbsoluteUrl(model.SearchIndexBaseUrl, "Base URL", required: true); + ValidateAbsoluteUrl(model.SearchIndexUrl, "Search index URL", required: false); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.AlgoliaSourceName, StringComparison.OrdinalIgnoreCase)) + { + if (string.IsNullOrWhiteSpace(model.AlgoliaApplicationId)) + { + _errors.Add("Application ID is required."); + } + + if (string.IsNullOrWhiteSpace(model.AlgoliaApiKey)) + { + _errors.Add("Search-only API key is required."); + } + + if (string.IsNullOrWhiteSpace(model.AlgoliaIndexName)) + { + _errors.Add("Index name is required."); + } + + return; + } + if (string.IsNullOrWhiteSpace(model.BaseUrl)) { _errors.Add("Base URL is required."); @@ -402,6 +521,45 @@ else private void Apply(AIToolInstanceViewModel model, AIToolInstance instance) { instance.Description = model.Description.Trim(); + instance.ModifiedUtc = TimeProvider.GetUtcNow().UtcDateTime; + + if (string.Equals(model.Source, DocumentationToolConstants.SitemapSourceName, StringComparison.OrdinalIgnoreCase)) + { + instance.Put(new SitemapDocumentationToolSettings + { + BaseUrl = model.SitemapBaseUrl?.Trim(), + SitemapUrl = string.IsNullOrWhiteSpace(model.SitemapUrl) ? null : model.SitemapUrl.Trim(), + MaxResults = model.SitemapMaxResults, + MaxPages = model.SitemapMaxPages, + }); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.SearchIndexSourceName, StringComparison.OrdinalIgnoreCase)) + { + instance.Put(new SearchIndexDocumentationToolSettings + { + BaseUrl = model.SearchIndexBaseUrl?.Trim(), + IndexUrl = string.IsNullOrWhiteSpace(model.SearchIndexUrl) ? null : model.SearchIndexUrl.Trim(), + MaxResults = model.SearchIndexMaxResults, + }); + + return; + } + + if (string.Equals(model.Source, DocumentationToolConstants.AlgoliaSourceName, StringComparison.OrdinalIgnoreCase)) + { + instance.Put(new AlgoliaDocumentationToolSettings + { + ApplicationId = model.AlgoliaApplicationId?.Trim(), + ApiKey = model.AlgoliaApiKey?.Trim(), + IndexName = model.AlgoliaIndexName?.Trim(), + MaxResults = model.AlgoliaMaxResults, + }); + + return; + } var protector = DataProtectionProvider.CreateProtector(HttpApiRequestToolConstants.DataProtectionPurpose); var existing = instance.TryGet(out var stored) ? stored : new HttpApiRequestToolSettings(); @@ -485,6 +643,47 @@ else : "{}"; } + if (instance.TryGet(out var sitemapSettings)) + { + model.SitemapBaseUrl = sitemapSettings.BaseUrl; + model.SitemapUrl = sitemapSettings.SitemapUrl; + model.SitemapMaxResults = sitemapSettings.MaxResults; + model.SitemapMaxPages = sitemapSettings.MaxPages; + } + + if (instance.TryGet(out var searchIndexSettings)) + { + model.SearchIndexBaseUrl = searchIndexSettings.BaseUrl; + model.SearchIndexUrl = searchIndexSettings.IndexUrl; + model.SearchIndexMaxResults = searchIndexSettings.MaxResults; + } + + if (instance.TryGet(out var algoliaSettings)) + { + model.AlgoliaApplicationId = algoliaSettings.ApplicationId; + model.AlgoliaApiKey = algoliaSettings.ApiKey; + model.AlgoliaIndexName = algoliaSettings.IndexName; + model.AlgoliaMaxResults = algoliaSettings.MaxResults; + } + return model; } + + private void ValidateAbsoluteUrl(string value, string label, bool required) + { + if (string.IsNullOrWhiteSpace(value)) + { + if (required) + { + _errors.Add($"{label} is required."); + } + + return; + } + + if (!Uri.TryCreate(value, UriKind.Absolute, out _)) + { + _errors.Add($"{label} must be a valid absolute URL."); + } + } } diff --git a/src/Startup/CrestApps.Core.Blazor.Web/Program.cs b/src/Startup/CrestApps.Core.Blazor.Web/Program.cs index 10a75d16..0de0fd07 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/Program.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/Program.cs @@ -13,6 +13,7 @@ using CrestApps.Core.AI.Elasticsearch; using CrestApps.Core.AI.Markdown; using CrestApps.Core.AI.Mcp; +using CrestApps.Core.AI.Tooling.Instances.Documentation; using CrestApps.Core.AI.Mcp.Ftp; using CrestApps.Core.AI.Mcp.Models; using CrestApps.Core.AI.Mcp.Sftp; @@ -104,6 +105,7 @@ ) .AddToolInstances(toolInstances => toolInstances .AddHttpApiRequestSource() + .AddDocumentationSearchSources() .AddEntityCoreStores() ) .AddDocumentProcessing(documentProcessing => documentProcessing diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIToolInstanceViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIToolInstanceViewModel.cs index 921ecb11..68876dd3 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIToolInstanceViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/AIToolInstanceViewModel.cs @@ -134,4 +134,62 @@ public sealed class AIToolInstanceViewModel /// Gets or sets an optional per-request timeout in seconds. /// public int? TimeoutSeconds { get; set; } + + /// + /// Gets or sets the base URL of the documentation site for the sitemap crawling source. + /// + public string SitemapBaseUrl { get; set; } + + /// + /// Gets or sets an explicit sitemap URL for the sitemap crawling source. When empty, the crawler + /// resolves the sitemap from the base URL by appending /sitemap.xml. + /// + public string SitemapUrl { get; set; } + + /// + /// Gets or sets the maximum number of results the sitemap source returns for a single search. + /// + public int? SitemapMaxResults { get; set; } + + /// + /// Gets or sets the maximum number of pages the sitemap crawler indexes for the site. + /// + public int? SitemapMaxPages { get; set; } + + /// + /// Gets or sets the base URL of the documentation site for the prebuilt search index source. + /// + public string SearchIndexBaseUrl { get; set; } + + /// + /// Gets or sets an explicit URL to the search index JSON for the prebuilt search index source. When + /// empty, the source resolves it from the base URL by appending /search/search_index.json. + /// + public string SearchIndexUrl { get; set; } + + /// + /// Gets or sets the maximum number of results the prebuilt search index source returns for a single search. + /// + public int? SearchIndexMaxResults { get; set; } + + /// + /// Gets or sets the Algolia application identifier for the Algolia DocSearch source. + /// + public string AlgoliaApplicationId { get; set; } + + /// + /// Gets or sets the Algolia search-only API key for the Algolia DocSearch source. This is a public, + /// client-safe key and is stored without additional protection. + /// + public string AlgoliaApiKey { get; set; } + + /// + /// Gets or sets the Algolia index name to query for the Algolia DocSearch source. + /// + public string AlgoliaIndexName { get; set; } + + /// + /// Gets or sets the maximum number of results the Algolia DocSearch source returns for a single search. + /// + public int? AlgoliaMaxResults { get; set; } } diff --git a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs index 880c4353..5bb4b24c 100644 --- a/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs +++ b/src/Startup/CrestApps.Core.Blazor.Web/ViewModels/SettingsViewModel.cs @@ -51,6 +51,14 @@ public sealed class SettingsViewModel public bool McpServerRequireAccessPermission { get; set; } = true; + public bool McpServerExposeAllTools { get; set; } + + public string[] McpServerSelectedToolNames { get; set; } = []; + + public List McpServerAvailableTools { get; set; } = []; + + public List McpServerAvailableToolInstances { get; set; } = []; + // Default deployment settings. public string DefaultChatDeploymentName { get; set; } @@ -161,3 +169,29 @@ public sealed class SettingsViewModel public List> AnthropicAvailableModels { get; set; } = []; } + +public sealed class McpServerToolSelectionItem +{ + public string Name { get; set; } + + public string Title { get; set; } + + public string Description { get; set; } + + public string Category { get; set; } + + public bool IsSelected { get; set; } +} + +public sealed class McpServerToolInstanceSelectionItem +{ + public string ItemId { get; set; } + + public string Name { get; set; } + + public string Description { get; set; } + + public string Source { get; set; } + + public bool IsSelected { get; set; } +} diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Controllers/SettingsController.cs b/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Controllers/SettingsController.cs index 0a22662a..9ee14cae 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Controllers/SettingsController.cs +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Controllers/SettingsController.cs @@ -12,17 +12,20 @@ using CrestApps.Core.AI.Profiles; using CrestApps.Core.AI.Security; using CrestApps.Core.AI.Speech; +using CrestApps.Core.AI.Tooling; using CrestApps.Core.Infrastructure.Indexing; using CrestApps.Core.Mvc.Web.Areas.Admin.ViewModels; using CrestApps.Core.Mvc.Web.Areas.AIChat.Models; using CrestApps.Core.Mvc.Web.Areas.AIChat.Services; using CrestApps.Core.Mvc.Web.Areas.ChatInteractions.Models; using CrestApps.Core.Mvc.Web.Models; +using CrestApps.Core.Services; using CrestApps.Core.Startup.Shared.Services; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Rendering; +using Microsoft.Extensions.Options; namespace CrestApps.Core.Mvc.Web.Areas.Admin.Controllers; @@ -41,6 +44,8 @@ public sealed class SettingsController : Controller private readonly IDataProtectionProvider _dataProtectionProvider; private readonly ISpeechVoiceResolver _speechVoiceResolver; private readonly ClaudeClientService _anthropicClientService; + private readonly AIToolDefinitionOptions _toolOptions; + private readonly ISourceCatalog _toolInstanceCatalog; public SettingsController( SiteSettingsStore siteSettings, @@ -49,7 +54,9 @@ public SettingsController( ISearchIndexProfileStore indexProfileStore, IDataProtectionProvider dataProtectionProvider, ISpeechVoiceResolver speechVoiceResolver, - ClaudeClientService anthropicClientService) + ClaudeClientService anthropicClientService, + IOptions toolOptions, + ISourceCatalog toolInstanceCatalog) { _siteSettings = siteSettings; _deploymentManager = deploymentManager; @@ -58,6 +65,8 @@ public SettingsController( _dataProtectionProvider = dataProtectionProvider; _speechVoiceResolver = speechVoiceResolver; _anthropicClientService = anthropicClientService; + _toolOptions = toolOptions.Value; + _toolInstanceCatalog = toolInstanceCatalog; } public async Task Index() @@ -111,6 +120,8 @@ public async Task Index() McpServerAuthenticationType = mcpServerSettings.AuthenticationType, McpServerApiKey = mcpServerSettings.ApiKey, McpServerRequireAccessPermission = mcpServerSettings.RequireAccessPermission, + McpServerExposeAllTools = mcpServerSettings.ExposeAllTools, + McpServerSelectedToolNames = mcpServerSettings.Tools?.ToArray() ?? [], CopilotAuthenticationType = copilotSettings.AuthenticationType, CopilotClientId = copilotSettings.ClientId, CopilotHasSecret = !string.IsNullOrWhiteSpace(copilotSettings.ProtectedClientSecret), @@ -148,6 +159,7 @@ public async Task Index() await NormalizeDeploymentSelectorsAsync(model); await PopulateDeploymentDropdownsAsync(model); await PopulateAdminWidgetProfilesAsync(model); + await PopulateMcpServerToolsAsync(model); await PopulateClaudeModelsAsync(model); PopulateBlockingThresholds(model); @@ -266,6 +278,7 @@ public async Task Save(SettingsViewModel model) { await PopulateDeploymentDropdownsAsync(model); await PopulateAdminWidgetProfilesAsync(model); + await PopulateMcpServerToolsAsync(model); await PopulateClaudeModelsAsync(model); PopulateBlockingThresholds(model); @@ -338,6 +351,14 @@ public async Task Save(SettingsViewModel model) AuthenticationType = model.McpServerAuthenticationType, ApiKey = model.McpServerApiKey?.Trim(), RequireAccessPermission = model.McpServerRequireAccessPermission, + ExposeAllTools = model.McpServerExposeAllTools, + Tools = model.McpServerExposeAllTools + ? [] + : (model.McpServerSelectedToolNames ?? []) + .Where(name => !string.IsNullOrWhiteSpace(name)) + .Select(name => name.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(), }); _siteSettings.Set(new MemoryMetadata @@ -481,6 +502,38 @@ private async Task PopulateAdminWidgetProfilesAsync(SettingsViewModel model) profile.ItemId == model.AdminWidgetProfileId)); } + private async Task PopulateMcpServerToolsAsync(SettingsViewModel model) + { + var selectedNames = new HashSet(model.McpServerSelectedToolNames ?? [], StringComparer.OrdinalIgnoreCase); + + model.McpServerAvailableTools = _toolOptions.Tools + .Where(tool => !tool.Value.Hidden) + .Select(tool => new McpServerToolSelectionItem + { + Name = tool.Key, + Title = tool.Value.Title ?? tool.Key, + Description = tool.Value.Description, + Category = tool.Value.Category ?? "Miscellaneous", + IsSelected = selectedNames.Contains(tool.Key) || selectedNames.Contains(tool.Value.Name), + }) + .OrderBy(tool => tool.Category, StringComparer.OrdinalIgnoreCase) + .ThenBy(tool => tool.Title, StringComparer.OrdinalIgnoreCase) + .ToList(); + + model.McpServerAvailableToolInstances = (await _toolInstanceCatalog.GetAllAsync()) + .Where(instance => !string.IsNullOrWhiteSpace(instance.Name)) + .OrderBy(instance => instance.Name, StringComparer.OrdinalIgnoreCase) + .Select(instance => new McpServerToolInstanceSelectionItem + { + ItemId = instance.ItemId, + Name = instance.Name, + Description = instance.Description, + Source = instance.Source, + IsSelected = selectedNames.Contains(instance.Name), + }) + .ToList(); + } + private async Task PopulateClaudeModelsAsync(SettingsViewModel model) { var settings = _siteSettings.Get(); diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/ViewModels/SettingsViewModel.cs b/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/ViewModels/SettingsViewModel.cs index 8d5b8c16..85fd5b91 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/ViewModels/SettingsViewModel.cs +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/ViewModels/SettingsViewModel.cs @@ -56,6 +56,16 @@ public sealed class SettingsViewModel public bool McpServerRequireAccessPermission { get; set; } = true; + public bool McpServerExposeAllTools { get; set; } + + public string[] McpServerSelectedToolNames { get; set; } = []; + + [BindNever] + public List McpServerAvailableTools { get; set; } = []; + + [BindNever] + public List McpServerAvailableToolInstances { get; set; } = []; + // Default deployment settings. public string DefaultChatDeploymentName { get; set; } @@ -184,3 +194,29 @@ public sealed class SettingsViewModel [BindNever] public IEnumerable BlockingThresholds { get; set; } = []; } + +public sealed class McpServerToolSelectionItem +{ + public string Name { get; set; } + + public string Title { get; set; } + + public string Description { get; set; } + + public string Category { get; set; } + + public bool IsSelected { get; set; } +} + +public sealed class McpServerToolInstanceSelectionItem +{ + public string ItemId { get; set; } + + public string Name { get; set; } + + public string Description { get; set; } + + public string Source { get; set; } + + public bool IsSelected { get; set; } +} diff --git a/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Views/Settings/Index.cshtml b/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Views/Settings/Index.cshtml index a3caf9b0..b022e21f 100644 --- a/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Views/Settings/Index.cshtml +++ b/src/Startup/CrestApps.Core.Mvc.Web/Areas/Admin/Views/Settings/Index.cshtml @@ -623,6 +623,79 @@
+
+

+ Choose which tools and configured tool instances are exposed to MCP clients. Nothing is + exposed by default; select the tools to allow, or expose everything. +

+
+ + +
+
+
Exposed tools
+ @if (Model.McpServerAvailableTools.Count > 0 || Model.McpServerAvailableToolInstances.Count > 0) + { +
+

Select the tools and configured tool instances MCP clients can discover and invoke.

+
+ + +
+
+ } + @if (Model.McpServerAvailableTools.Count == 0) + { +
No AI tools are registered.
+ } + else + { + @foreach (var group in Model.McpServerAvailableTools.GroupBy(tool => tool.Category).OrderBy(group => group.Key)) + { +
@group.Key
+ @foreach (var tool in group) + { +
+ + +
+ } + } + } + +
Exposed tool instances
+ @if (Model.McpServerAvailableToolInstances.Count == 0) + { +
No tool instances are configured. Add them under AI Tool Instances first.
+ } + else + { + @foreach (var instance in Model.McpServerAvailableToolInstances) + { +
+ + +
+ } + } +
@@ -861,6 +934,41 @@ loadVoiceOptions(); }); + + @await RenderSectionAsync("Scripts", required: false) @await Html.PartialAsync("_ToastNotifications") diff --git a/tests/CrestApps.Core.Tests/Core/Mcp/McpServerBuilderExtensionsTests.cs b/tests/CrestApps.Core.Tests/Core/Mcp/McpServerBuilderExtensionsTests.cs index a6336e9a..cbedafe1 100644 --- a/tests/CrestApps.Core.Tests/Core/Mcp/McpServerBuilderExtensionsTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Mcp/McpServerBuilderExtensionsTests.cs @@ -2,31 +2,92 @@ using CrestApps.Core.AI.Mcp; using CrestApps.Core.AI.Mcp.Services; using CrestApps.Core.AI.Tooling; +using CrestApps.Core.Services; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ModelContextProtocol; using ModelContextProtocol.Protocol; using ModelContextProtocol.Server; using Moq; +using ServerToolOptions = CrestApps.Core.AI.Mcp.Models.McpServerOptions; namespace CrestApps.Core.Tests.Core.Mcp; public sealed class McpServerBuilderExtensionsTests { /// - /// Verifies that visible local tools retain registration order, precede SDK tools, and hidden tools are omitted. + /// Verifies that with the default settings (nothing allowed and ExposeAllTools off) no tools + /// are listed, even when tools are registered, so an MCP server exposes nothing until explicitly told to. /// [Fact] - public async Task ListToolsHandler_ReturnsVisibleLocalToolsFirstAndOmitsHiddenTools() + public async Task ListToolsHandler_DefaultDeny_ReturnsEmpty() { - var services = CreateServices( - CreateSdkTool("sdk-first"), - CreateSdkTool("sdk-second")); + var services = CreateServices(); + + AddLocalTool(services, "search-key", new TestAIFunction("search")); + AddLocalTool(services, "create-key", new TestAIFunction("create")); - AddLocalTool(services, "local-first-key", new TestAIFunction("local-first")); + using var serviceProvider = services.BuildServiceProvider(); + + var result = await InvokeListToolsHandlerAsync( + serviceProvider, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Empty(result.Tools); + } + + /// + /// Verifies that enabling ExposeAllTools lists every non-hidden tool while hidden tools are omitted. + /// + [Fact] + public async Task ListToolsHandler_ExposeAllTools_ReturnsVisibleToolsAndOmitsHidden() + { + var services = CreateServices(configureOptions: options => options.ExposeAllTools = true); + + AddLocalTool(services, "search-key", new TestAIFunction("search")); AddLocalTool(services, "hidden-key", new TestAIFunction("hidden"), hidden: true); - AddLocalTool(services, "local-second-key", new TestAIFunction("local-second")); + AddLocalTool(services, "create-key", new TestAIFunction("create")); + + using var serviceProvider = services.BuildServiceProvider(); + + var result = await InvokeListToolsHandlerAsync( + serviceProvider, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(["search", "create"], result.Tools.Select(tool => tool.Name)); + } + + /// + /// Verifies that the allow-list exposes only the tools whose registration key is listed. + /// + [Fact] + public async Task ListToolsHandler_AllowList_ExposesOnlyNamedTools() + { + var services = CreateServices(configureOptions: options => options.Tools = ["search-key"]); + + AddLocalTool(services, "search-key", new TestAIFunction("search")); + AddLocalTool(services, "create-key", new TestAIFunction("create")); + + using var serviceProvider = services.BuildServiceProvider(); + + var result = await InvokeListToolsHandlerAsync( + serviceProvider, + cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(["search"], result.Tools.Select(tool => tool.Name)); + } + + /// + /// Verifies that allow-list matching is case-insensitive. + /// + [Fact] + public async Task ListToolsHandler_AllowList_MatchesNameCaseInsensitively() + { + var services = CreateServices(configureOptions: options => options.Tools = ["SEARCH-KEY"]); + + AddLocalTool(services, "search-key", new TestAIFunction("search")); using var serviceProvider = services.BuildServiceProvider(); @@ -34,18 +95,16 @@ public async Task ListToolsHandler_ReturnsVisibleLocalToolsFirstAndOmitsHiddenTo serviceProvider, cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal( - ["local-first", "local-second", "sdk-first", "sdk-second"], - result.Tools.Select(tool => tool.Name)); + Assert.Equal(["search"], result.Tools.Select(tool => tool.Name)); } /// - /// Verifies that keyed local tool creation failures are logged and skipped. + /// Verifies that keyed tool creation failures are logged and skipped instead of failing the whole list. /// [Fact] public async Task ListToolsHandler_LogsAndSkipsKeyedServiceCreationFailures() { - var services = CreateServices(); + var services = CreateServices(configureOptions: options => options.ExposeAllTools = true); var logger = new Mock>(); var failure = new InvalidOperationException("Tool creation failed."); @@ -77,15 +136,22 @@ public async Task ListToolsHandler_LogsAndSkipsKeyedServiceCreationFailures() } /// - /// Verifies that duplicate protocol names produced by distinct local registrations remain in the result. + /// Verifies that a configured tool instance is exposed when its name is on the allow-list. /// [Fact] - public async Task ListToolsHandler_PreservesDuplicateNamesProducedByLocalTools() + public async Task ListToolsHandler_AllowList_ExposesNamedToolInstance() { - var services = CreateServices(CreateSdkTool("sdk")); + var instance = new AIToolInstance + { + ItemId = "instance-1", + Source = "docs-source", + Name = "crestapps-docs", + }; - AddLocalTool(services, "first-key", new TestAIFunction("duplicate")); - AddLocalTool(services, "second-key", new TestAIFunction("duplicate")); + var services = CreateServices(configureOptions: options => options.Tools = ["crestapps-docs"]); + + AddToolInstances(services, instance); + AddToolInstanceSource(services, "docs-source"); using var serviceProvider = services.BuildServiceProvider(); @@ -93,19 +159,26 @@ public async Task ListToolsHandler_PreservesDuplicateNamesProducedByLocalTools() serviceProvider, cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(["duplicate", "duplicate", "sdk"], result.Tools.Select(tool => tool.Name)); + Assert.Equal(["crestapps-docs"], result.Tools.Select(tool => tool.Name)); } /// - /// Verifies that SDK tools are appended in service enumeration order. + /// Verifies that configured tool instances are exposed when ExposeAllTools is enabled. /// [Fact] - public async Task ListToolsHandler_AppendsSdkToolsInEnumerationOrder() + public async Task ListToolsHandler_ExposeAll_IncludesToolInstances() { - var services = CreateServices( - CreateSdkTool("sdk-third"), - CreateSdkTool("sdk-first"), - CreateSdkTool("sdk-second")); + var instance = new AIToolInstance + { + ItemId = "instance-1", + Source = "docs-source", + Name = "crestapps-docs", + }; + + var services = CreateServices(configureOptions: options => options.ExposeAllTools = true); + + AddToolInstances(services, instance); + AddToolInstanceSource(services, "docs-source"); using var serviceProvider = services.BuildServiceProvider(); @@ -113,25 +186,26 @@ public async Task ListToolsHandler_AppendsSdkToolsInEnumerationOrder() serviceProvider, cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal( - ["sdk-third", "sdk-first", "sdk-second"], - result.Tools.Select(tool => tool.Name)); + Assert.Contains(result.Tools, tool => tool.Name == "crestapps-docs"); } /// - /// Verifies that a local tool takes precedence over an SDK tool with the same exact name. + /// Verifies that configured tool instances are not exposed by default. /// [Fact] - public async Task ListToolsHandler_SkipsSdkToolsThatDuplicateLocalNames() + public async Task ListToolsHandler_DefaultDeny_OmitsToolInstances() { - var duplicateSdkTool = CreateSdkTool("duplicate", "SDK duplicate"); - var services = CreateServices(duplicateSdkTool, CreateSdkTool("sdk")); - var localDescription = "Local duplicate"; + var instance = new AIToolInstance + { + ItemId = "instance-1", + Source = "docs-source", + Name = "crestapps-docs", + }; + + var services = CreateServices(); - AddLocalTool( - services, - "local-key", - new TestAIFunction("duplicate", localDescription)); + AddToolInstances(services, instance); + AddToolInstanceSource(services, "docs-source"); using var serviceProvider = services.BuildServiceProvider(); @@ -139,168 +213,190 @@ public async Task ListToolsHandler_SkipsSdkToolsThatDuplicateLocalNames() serviceProvider, cancellationToken: TestContext.Current.CancellationToken); - Assert.Equal(["duplicate", "sdk"], result.Tools.Select(tool => tool.Name)); - Assert.Equal(localDescription, result.Tools[0].Description); - Assert.DoesNotContain(result.Tools, tool => ReferenceEquals(tool, duplicateSdkTool.ProtocolTool)); + Assert.Empty(result.Tools); } /// - /// Verifies that later SDK tools with duplicate names are skipped while the first instance is retained. + /// Verifies that a tool not on the allow-list cannot be invoked through the call handler. /// [Fact] - public async Task ListToolsHandler_SkipsDuplicateNamesWithinSdkTools() + public async Task CallToolHandler_DefaultDeny_RejectsTool() { - var first = CreateSdkTool("duplicate", "first"); - var second = CreateSdkTool("duplicate", "second"); - var unique = CreateSdkTool("unique"); - var services = CreateServices(first, second, unique); + var services = CreateServices(); - using var serviceProvider = services.BuildServiceProvider(); + AddLocalTool(services, "search", new TestAIFunction("search")); - var result = await InvokeListToolsHandlerAsync( - serviceProvider, - cancellationToken: TestContext.Current.CancellationToken); + using var serviceProvider = services.BuildServiceProvider(); - Assert.Equal(["duplicate", "unique"], result.Tools.Select(tool => tool.Name)); - Assert.Same(first.ProtocolTool, result.Tools[0]); - Assert.DoesNotContain(result.Tools, tool => ReferenceEquals(tool, second.ProtocolTool)); + await Assert.ThrowsAsync(async () => + await InvokeCallToolHandlerAsync( + serviceProvider, + "search", + TestContext.Current.CancellationToken)); } /// - /// Verifies that duplicate matching uses ordinal case-sensitive equality. + /// Verifies that an allow-listed tool can be invoked through the call handler. /// [Fact] - public async Task ListToolsHandler_TreatsToolNamesAsOrdinalCaseSensitive() + public async Task CallToolHandler_InvokesAllowedTool() { - var services = CreateServices( - CreateSdkTool("casetool"), - CreateSdkTool("CaseTool")); + var services = CreateServices(configureOptions: options => options.Tools = ["search"]); - AddLocalTool(services, "local-key", new TestAIFunction("CaseTool")); + AddLocalTool(services, "search", new TestAIFunction("search")); using var serviceProvider = services.BuildServiceProvider(); - var result = await InvokeListToolsHandlerAsync( + var result = await InvokeCallToolHandlerAsync( serviceProvider, - cancellationToken: TestContext.Current.CancellationToken); + "search", + TestContext.Current.CancellationToken); - Assert.Equal(["CaseTool", "casetool"], result.Tools.Select(tool => tool.Name)); + Assert.NotNull(result); } /// - /// Verifies that a service provider returning a null SDK tool enumerable produces an empty result. + /// Verifies that a tool advertised by its function name can be invoked even when the function name + /// differs from the registration key it is keyed under, mirroring how the list handler publishes the + /// function name rather than the key. /// [Fact] - public async Task ListToolsHandler_AllowsNullSdkToolEnumerable() + public async Task CallToolHandler_InvokesTool_WhenFunctionNameDiffersFromKey() { - var services = CreateServices(); + var services = CreateServices(configureOptions: options => options.Tools = ["search-key"]); + + AddLocalTool(services, "search-key", new TestAIFunction("search")); using var serviceProvider = services.BuildServiceProvider(); - var requestServices = new NullSdkEnumerableServiceProvider(serviceProvider); - var result = await InvokeListToolsHandlerAsync( + var result = await InvokeCallToolHandlerAsync( serviceProvider, - requestServices, + "search", TestContext.Current.CancellationToken); - Assert.Empty(result.Tools); + Assert.NotNull(result); } - - /// - /// Verifies that the default DI empty SDK tool enumerable leaves local tools unchanged. - /// [Fact] - public async Task ListToolsHandler_AllowsDefaultEmptySdkToolEnumerable() + public async Task CallToolHandler_ExposeAll_InvokesTool() { - var services = CreateServices(); + var services = CreateServices(configureOptions: options => options.ExposeAllTools = true); - AddLocalTool(services, "local-key", new TestAIFunction("local")); + AddLocalTool(services, "search", new TestAIFunction("search")); using var serviceProvider = services.BuildServiceProvider(); - var sdkTools = serviceProvider.GetRequiredService>(); - - Assert.Empty(sdkTools); - var result = await InvokeListToolsHandlerAsync( + var result = await InvokeCallToolHandlerAsync( serviceProvider, - cancellationToken: TestContext.Current.CancellationToken); + "search", + TestContext.Current.CancellationToken); - Assert.Equal(["local"], result.Tools.Select(tool => tool.Name)); + Assert.NotNull(result); } /// - /// Verifies that local metadata values and SDK protocol tool instances retain their identity. + /// Verifies that an allow-listed tool instance can be invoked through the call handler by its name. /// [Fact] - public async Task ListToolsHandler_PreservesToolSchemaDescriptionAndSdkProtocolIdentity() + public async Task CallToolHandler_InvokesAllowedToolInstance() { - var schema = JsonSerializer.Deserialize( - """ + var instance = new AIToolInstance { - "type": "object", - "properties": { - "value": { - "type": "integer" - } - } - } - """); - var description = new string("Local description".ToCharArray()); - var localTool = new TestAIFunction("local", description, schema); - var sdkTool = CreateSdkTool("sdk"); - var services = CreateServices(sdkTool); + ItemId = "instance-1", + Source = "docs-source", + Name = "crestapps-docs", + }; + + var services = CreateServices(configureOptions: options => options.Tools = ["crestapps-docs"]); - AddLocalTool(services, "local-key", localTool); + AddToolInstances(services, instance); + AddToolInstanceSource(services, "docs-source"); using var serviceProvider = services.BuildServiceProvider(); - var result = await InvokeListToolsHandlerAsync( + var result = await InvokeCallToolHandlerAsync( serviceProvider, - cancellationToken: TestContext.Current.CancellationToken); + "crestapps-docs", + TestContext.Current.CancellationToken); - Assert.Same(description, result.Tools[0].Description); - Assert.Equal(localTool.JsonSchema, result.Tools[0].InputSchema); - Assert.Same(sdkTool.ProtocolTool, result.Tools[1]); + Assert.NotNull(result); } /// - /// Verifies the current synchronous list handler behavior when passed an already-canceled token. + /// Verifies that a tool instance not on the allow-list cannot be invoked through the call handler. /// [Fact] - public async Task ListToolsHandler_DoesNotObserveCancellation() + public async Task CallToolHandler_DefaultDeny_RejectsToolInstance() { - var services = CreateServices(CreateSdkTool("sdk")); + var instance = new AIToolInstance + { + ItemId = "instance-1", + Source = "docs-source", + Name = "crestapps-docs", + }; - AddLocalTool(services, "local-key", new TestAIFunction("local")); + var services = CreateServices(); - using var serviceProvider = services.BuildServiceProvider(); - using var cancellationTokenSource = new CancellationTokenSource(); - cancellationTokenSource.Cancel(); + AddToolInstances(services, instance); + AddToolInstanceSource(services, "docs-source"); - var result = await InvokeListToolsHandlerAsync( - serviceProvider, - cancellationToken: cancellationTokenSource.Token); + using var serviceProvider = services.BuildServiceProvider(); - Assert.Equal(["local", "sdk"], result.Tools.Select(tool => tool.Name)); + await Assert.ThrowsAsync(async () => + await InvokeCallToolHandlerAsync( + serviceProvider, + "crestapps-docs", + TestContext.Current.CancellationToken)); } /// /// Creates the MCP service collection and registers the CrestApps handlers. /// - /// The SDK tools to register in enumeration order. + /// An optional delegate that configures the exposure settings. /// The configured service collection. - private static ServiceCollection CreateServices(params McpServerTool[] sdkTools) + private static ServiceCollection CreateServices( + Action configureOptions = null) { var services = new ServiceCollection(); var builder = services.AddMcpServer(); services.AddOptions(); - builder.WithTools(sdkTools); + services.AddOptions(); + + if (configureOptions is not null) + { + services.Configure(configureOptions); + } + builder.WithCrestAppsHandlers(); return services; } + /// + /// Registers a fake tool instance catalog returning the supplied instances. + /// + /// The service collection. + /// The instances to return. + private static void AddToolInstances(IServiceCollection services, params AIToolInstance[] instances) + { + var catalog = new Mock>(); + catalog + .Setup(value => value.GetAllAsync(It.IsAny())) + .ReturnsAsync(instances); + + services.AddSingleton(catalog.Object); + } + + /// + /// Registers a keyed tool instance source that produces a function named after the instance. + /// + /// The service collection. + /// The registered source name. + private static void AddToolInstanceSource(IServiceCollection services, string sourceName) + { + services.AddKeyedSingleton(sourceName, (_, _) => new TestToolInstanceSource()); + } + /// /// Registers a local tool definition and keyed tool instance. /// @@ -377,48 +473,52 @@ private static async ValueTask InvokeListToolsHandlerAsync( } /// - /// Creates an SDK MCP tool with the supplied protocol metadata. + /// Invokes the registered CrestApps call-tool handler. /// - /// The tool name. - /// The optional description. - /// The SDK MCP tool. - private static McpServerTool CreateSdkTool(string name, string description = null) + /// The provider containing the registered handler. + /// The name of the tool to invoke. + /// The cancellation token passed to the handler. + /// The call-tool result. + private static async ValueTask InvokeCallToolHandlerAsync( + IServiceProvider serviceProvider, + string toolName, + CancellationToken cancellationToken = default) { - return McpServerTool.Create( - (Func)(static () => string.Empty), - new McpServerToolCreateOptions - { - Name = name, - Description = description, - }); - } + var options = serviceProvider.GetRequiredService>().Value; + var handler = options.Handlers.CallToolHandler; + var server = new Mock(); - private sealed class NullSdkEnumerableServiceProvider : IServiceProvider - { - private readonly IServiceProvider _serviceProvider; + Assert.NotNull(handler); + server.SetupGet(instance => instance.Services).Returns(serviceProvider); - /// - /// Initializes a provider that masks the SDK tool enumerable. - /// - /// The underlying service provider. - public NullSdkEnumerableServiceProvider(IServiceProvider serviceProvider) + var request = new RequestContext( + server.Object, + new JsonRpcRequest + { + Method = RequestMethods.ToolsCall, + Id = new RequestId("1"), + }, + new CallToolRequestParams + { + Name = toolName, + }) { - _serviceProvider = serviceProvider; - } + Services = serviceProvider, + }; + + return await handler(request, cancellationToken); + } + private sealed class TestToolInstanceSource : IAIToolInstanceSource + { /// - /// Resolves a service, returning null for the SDK tool enumerable. + /// Creates a function whose name mirrors the instance name so tests can assert on it. /// - /// The service type. - /// The resolved service. - public object GetService(Type serviceType) + /// The configured instance. + /// The produced function. + public AITool CreateTool(AIToolInstance instance) { - if (serviceType == typeof(IEnumerable)) - { - return null; - } - - return _serviceProvider.GetService(serviceType); + return new TestAIFunction(instance.Name); } } diff --git a/tests/CrestApps.Core.Tests/Core/Tools/DocumentationSearchTests.cs b/tests/CrestApps.Core.Tests/Core/Tools/DocumentationSearchTests.cs new file mode 100644 index 00000000..7b0cced5 --- /dev/null +++ b/tests/CrestApps.Core.Tests/Core/Tools/DocumentationSearchTests.cs @@ -0,0 +1,221 @@ +using CrestApps.Core.AI.Tooling.Instances.Documentation; +using CrestApps.Core.AI.Tooling; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace CrestApps.Core.Tests.Core.Tools; + +public sealed class DocumentationSearchTests +{ + /// + /// Verifies that the sitemap source materializes a documentation search function whose model-facing + /// name and description are derived from the configured instance. + /// + [Fact] + public void SitemapSource_CreateTool_DerivesNameAndDescriptionFromInstance() + { + var instance = new AIToolInstance + { + ItemId = "instance-1", + Source = DocumentationToolConstants.SitemapSourceName, + Name = "crestapps-docs", + Description = "Searches the CrestApps documentation.", + }; + + instance.Put(new SitemapDocumentationToolSettings + { + BaseUrl = "https://core.crestapps.com", + }); + + var source = new SitemapDocumentationToolSource(); + var tool = source.CreateTool(instance); + + var function = Assert.IsType(tool); + + Assert.Equal(instance.GetFunctionName(), function.Name); + Assert.Equal("Searches the CrestApps documentation.", function.Description); + } + + /// + /// Verifies that the search-index source produces a documentation search function. + /// + [Fact] + public void SearchIndexSource_CreateTool_ProducesFunction() + { + var instance = new AIToolInstance + { + ItemId = "instance-2", + Source = DocumentationToolConstants.SearchIndexSourceName, + Name = "mkdocs", + }; + + instance.Put(new SearchIndexDocumentationToolSettings + { + BaseUrl = "https://docs.example.com", + IndexUrl = "https://docs.example.com/search/search_index.json", + }); + + var source = new SearchIndexDocumentationToolSource(); + var tool = source.CreateTool(instance); + + Assert.IsType(tool); + } + + /// + /// Verifies that the Algolia source produces a documentation search function. + /// + [Fact] + public void AlgoliaSource_CreateTool_ProducesFunction() + { + var instance = new AIToolInstance + { + ItemId = "instance-3", + Source = DocumentationToolConstants.AlgoliaSourceName, + Name = "algolia", + }; + + instance.Put(new AlgoliaDocumentationToolSettings + { + ApplicationId = "APP123", + ApiKey = "search-key", + IndexName = "docs-index", + }); + + var source = new AlgoliaDocumentationToolSource(); + var tool = source.CreateTool(instance); + + Assert.IsType(tool); + } + + /// + /// Verifies that invoking the function returns a helpful message when the required query is missing. + /// + [Fact] + public async Task InvokeAsync_WhenQueryMissing_ReturnsMessage() + { + var function = CreateFunction(new FakeDocumentationSource("docs")); + + var result = await InvokeAsync(function, query: null); + + Assert.Contains("query", result, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Verifies that invoking the function returns a message when the source yields no results. + /// + [Fact] + public async Task InvokeAsync_WhenNoResults_ReturnsMessage() + { + var function = CreateFunction(new FakeDocumentationSource("docs")); + + var result = await InvokeAsync(function, "anything"); + + Assert.Equal("No documentation results were found for 'anything'.", result); + } + + /// + /// Verifies that invoking the function formats the results returned by the bound source. + /// + [Fact] + public async Task InvokeAsync_FormatsResults() + { + var source = new FakeDocumentationSource( + "docs", + new DocumentationSearchResult + { + SourceName = "docs", + Title = "Getting started", + Url = "https://core.crestapps.com/start", + Snippet = "How to begin.", + Score = 5, + }); + + var function = CreateFunction(source); + + var result = await InvokeAsync(function, "start"); + + Assert.Contains("[1] Getting started — https://core.crestapps.com/start", result); + Assert.Contains("How to begin.", result); + } + + /// + /// Verifies that the materializer caches the built source until the signature changes. + /// + [Fact] + public void Materializer_CachesUntilSignatureChanges() + { + var materializer = new DefaultDocumentationSourceMaterializer(); + var buildCount = 0; + + IDocumentationSource Factory() + { + buildCount++; + + return new FakeDocumentationSource("docs"); + } + + var first = materializer.GetOrCreate("key", "sig-1", Factory); + var second = materializer.GetOrCreate("key", "sig-1", Factory); + + Assert.Same(first, second); + Assert.Equal(1, buildCount); + + var third = materializer.GetOrCreate("key", "sig-2", Factory); + + Assert.NotSame(first, third); + Assert.Equal(2, buildCount); + } + + private static DocumentationSearchToolFunction CreateFunction(IDocumentationSource source) + { + var instance = new AIToolInstance + { + ItemId = "instance-1", + Name = "docs", + CreatedUtc = DateTime.UnixEpoch, + }; + + return new DocumentationSearchToolFunction("docs", "Docs search", instance, _ => source); + } + + private static async Task InvokeAsync(DocumentationSearchToolFunction function, string query) + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + + var arguments = new AIFunctionArguments + { + Services = provider, + }; + + if (query is not null) + { + arguments["query"] = query; + } + + var result = await function.InvokeAsync(arguments, TestContext.Current.CancellationToken); + + return result?.ToString(); + } + + private sealed class FakeDocumentationSource : IDocumentationSource + { + private readonly DocumentationSearchResult[] _results; + + public FakeDocumentationSource(string name, params DocumentationSearchResult[] results) + { + Name = name; + _results = results; + } + + public string Name { get; } + + public Task> SearchAsync(DocumentationSearchRequest request, CancellationToken cancellationToken) + { + return Task.FromResult>(_results); + } + } +}