Skip to content

Fix Kubernetes hostname publishing and routing #19430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Mitch Denny (mitchdenny) wants to merge 1 commit into
base: main
Choose a base branch
from
+402 −78
Open

Fix Kubernetes hostname publishing and routing #19430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 121 additions & 70 deletions src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,8 +580,8 @@ private static async Task<string> ResolveExpressionAtDeployTimeAsync(ReferenceEx
/// <summary>
/// Resolves a <see cref="ReferenceExpression"/> for inclusion in a Kubernetes manifest
/// produced by an ingress or gateway resource. When the expression wraps one or more
/// <see cref="ParameterResource"/> instances that have no value at publish time
/// (e.g., user-supplied parameters without defaults, or secrets), the expression is
/// <see cref="ParameterResource"/> instances that must remain deploy-time inputs
/// (for example, secrets or parameters without published defaults), the expression is
/// rendered with Helm template placeholders (such as <c>{{ .Values.parameters.ingress.ingressclass }}</c>)
/// and the parameters are captured for deploy-time resolution into a values override file.
/// </summary>
Expand All @@ -594,31 +594,33 @@ private static async Task<string> ResolveExpressionAtDeployTimeAsync(ReferenceEx
/// <param name="cancellationToken">The cancellation token.</param>
private async Task<string> ResolveExpressionAsync(ReferenceExpression expression, string owningResourceName, CancellationToken cancellationToken)
{
try
{
return (await expression.GetValueAsync(cancellationToken).ConfigureAwait(false))!;
}
catch (MissingParameterValueException)
if (!expression.ValueProviders.OfType<ParameterResource>().Any(ShouldCaptureAsHelmValue))
{
// One or more parameters in the expression have no value at publish time
// (e.g., a parameter created via AddParameter("ingressclass") with no default,
// or a secret parameter). Substitute each unresolved parameter with a Helm
// template reference into values.yaml so the resulting manifest is a valid
// Helm template and the value can be supplied at deploy time.
var owningResourceKey = owningResourceName.ToHelmValuesSectionName();
var args = new object[expression.ValueProviders.Count];

for (var i = 0; i < expression.ValueProviders.Count; i++)
try
{
return (await expression.GetValueAsync(cancellationToken).ConfigureAwait(false))!;
}
catch (MissingParameterValueException)
{
args[i] = await ResolveValueProviderAsync(
expression.ValueProviders[i],
owningResourceName,
owningResourceKey,
cancellationToken).ConfigureAwait(false);
// Fall through to Helm-reference substitution below.
}
}

return string.Format(System.Globalization.CultureInfo.InvariantCulture, expression.Format, args);
// Parameters without published defaults may still have runtime values, but publish must
// preserve them as deploy-time inputs instead of leaking those values into generated YAML.
var owningResourceKey = owningResourceName.ToHelmValuesSectionName();
var args = new object[expression.ValueProviders.Count];

for (var i = 0; i < expression.ValueProviders.Count; i++)
{
args[i] = await ResolveValueProviderAsync(
expression.ValueProviders[i],
owningResourceName,
owningResourceKey,
cancellationToken).ConfigureAwait(false);
}

return string.Format(System.Globalization.CultureInfo.InvariantCulture, expression.Format, args);
}

private async Task<string> ResolveValueProviderAsync(
Expand All @@ -629,19 +631,16 @@ private async Task<string> ResolveValueProviderAsync(
{
if (valueProvider is ParameterResource parameter)
{
// Attempt to resolve this individual parameter first. The outer
// MissingParameterValueException from the whole-expression resolve attempt
// only tells us that *some* parameter in the expression was unresolved;
// others (e.g., those with `publishValueAsDefault: true`) may still have
// a value and should be inlined into the manifest rather than left as
// Helm placeholders. This keeps the published chart maximally self-contained.
try
if (!ShouldCaptureAsHelmValue(parameter))
{
return (await parameter.GetValueAsync(cancellationToken).ConfigureAwait(false)) ?? string.Empty;
}
catch (MissingParameterValueException)
{
// Fall through to Helm-reference substitution below.
try
{
return (await parameter.GetValueAsync(cancellationToken).ConfigureAwait(false)) ?? string.Empty;
}
catch (MissingParameterValueException)
{
// Fall through to Helm-reference substitution below.
}
}

// Capture the parameter so HelmDeploymentEngine writes its resolved value to
Expand Down Expand Up @@ -673,6 +672,24 @@ private async Task<string> ResolveValueProviderAsync(
return (await valueProvider.GetValueAsync(cancellationToken).ConfigureAwait(false)) ?? string.Empty;
}

private static bool ShouldCaptureAsHelmValue(ParameterResource parameter)
=> parameter.Secret || parameter.Default is null;

private async Task<List<string>> ResolveHostnamesAsync(
IEnumerable<ReferenceExpression> hostnames,
string owningResourceName,
CancellationToken cancellationToken)
{
var resolvedHostnames = new List<string>();

foreach (var hostname in hostnames)
{
resolvedHostnames.Add(await ResolveExpressionAsync(hostname, owningResourceName, cancellationToken).ConfigureAwait(false));
}

return resolvedHostnames;
}

private async Task ProcessIngressResources(DistributedApplicationModel model, Dictionary<IResource, KubernetesResource> deploymentTargets, ILogger logger, CancellationToken cancellationToken)
{
var ingressResources = model.Resources
Expand Down Expand Up @@ -735,17 +752,40 @@ private async Task ProcessIngressResources(DistributedApplicationModel model, Di
ingress.Metadata.Annotations[key] = await ResolveExpressionAsync(value, ingressResource.Name, cancellationToken).ConfigureAwait(false);
}

var pathsByHost = ingressResource.Paths.GroupBy(p => p.Host ?? string.Empty);
var resolvedHostnames = await ResolveHostnamesAsync(
ingressResource.Hostnames,
ingressResource.Name,
cancellationToken).ConfigureAwait(false);
Comment thread
mitchdenny marked this conversation as resolved.
var pathsByHost = new Dictionary<string, List<IngressPathConfig>>();

foreach (var path in ingressResource.Paths)
{
if (path.Host is { } explicitHost)
{
AddPathForHost(explicitHost, path);
}
else if (resolvedHostnames.Count == 0)
{
AddPathForHost(string.Empty, path);
}
else
{
foreach (var hostname in resolvedHostnames)
{
AddPathForHost(hostname, path);
}
}
}

foreach (var hostGroup in pathsByHost)
foreach (var (host, paths) in pathsByHost)
{
var rule = new IngressRuleV1();
if (!string.IsNullOrEmpty(hostGroup.Key))
if (!string.IsNullOrEmpty(host))
{
rule.Host = hostGroup.Key;
rule.Host = host;
}

foreach (var pathRule in hostGroup)
foreach (var pathRule in paths)
{
var backend = ResolveIngressBackend(pathRule.Endpoint, deploymentTargets, ingressResource.Name, logger);
if (backend is null)
Expand Down Expand Up @@ -783,58 +823,52 @@ private async Task ProcessIngressResources(DistributedApplicationModel model, Di
SecretName = await ResolveExpressionAsync(tls.SecretName, ingressResource.Name, cancellationToken).ConfigureAwait(false),
};

foreach (var host in ingressResource.Hostnames)
{
tlsEntry.Hosts.Add(await ResolveExpressionAsync(host, ingressResource.Name, cancellationToken).ConfigureAwait(false));
}
tlsEntry.Hosts.AddRange(resolvedHostnames);

ingress.Spec.Tls.Add(tlsEntry);
}

// Auto-generate rules for TLS hosts that don't have explicit routes.
if (ingress.Spec.DefaultBackend is not null)
// A default backend remains catch-all even when hostnames are configured. Only synthesize
// host rules for TLS because some ingress controllers require each TLS host to have a rule.
if (ingress.Spec.DefaultBackend is not null && ingress.Spec.Tls.Count > 0)
{
var hostsWithRules = new HashSet<string>(
ingress.Spec.Rules
.Where(r => r.Host is not null)
.Select(r => r.Host!),
StringComparer.OrdinalIgnoreCase);

foreach (var tls in ingressResource.TlsConfigs)
foreach (var resolvedHost in resolvedHostnames)
{
foreach (var host in ingressResource.Hostnames)
if (hostsWithRules.Add(resolvedHost))
{
var resolvedHost = await ResolveExpressionAsync(host, ingressResource.Name, cancellationToken).ConfigureAwait(false);
if (!hostsWithRules.Contains(resolvedHost))
ingress.Spec.Rules.Add(new IngressRuleV1
{
ingress.Spec.Rules.Add(new IngressRuleV1
Host = resolvedHost,
Http = new HttpIngressRuleValueV1
{
Host = resolvedHost,
Http = new HttpIngressRuleValueV1
Paths =
{
Paths =
new HttpIngressPathV1
{
new HttpIngressPathV1
Path = "/",
PathType = IngressPathType.Prefix.ToKubernetesString(),
Backend = new IngressBackendV1
{
Path = "/",
PathType = IngressPathType.Prefix.ToKubernetesString(),
Backend = new IngressBackendV1
Service = new IngressServiceBackendV1
{
Service = new IngressServiceBackendV1
Name = ingress.Spec.DefaultBackend.Service.Name,
Port = new ServiceBackendPortV1
{
Name = ingress.Spec.DefaultBackend.Service.Name,
Port = new ServiceBackendPortV1
{
Name = ingress.Spec.DefaultBackend.Service.Port.Name,
Number = ingress.Spec.DefaultBackend.Service.Port.Number
}
Name = ingress.Spec.DefaultBackend.Service.Port.Name,
Number = ingress.Spec.DefaultBackend.Service.Port.Number
}
}
}
}
}
});
}
}
});
}
}
}
Expand All @@ -846,6 +880,17 @@ private async Task ProcessIngressResources(DistributedApplicationModel model, Di
}

return ingress;

void AddPathForHost(string host, IngressPathConfig path)
{
if (!pathsByHost.TryGetValue(host, out var paths))
{
paths = [];
pathsByHost.Add(host, paths);
}

paths.Add(path);
}
}

private static IngressBackendV1? ResolveIngressBackend(
Expand Down Expand Up @@ -1000,6 +1045,10 @@ private async Task BuildGatewayObjects(
{
Metadata = { Name = gatewayName }
};
var resolvedHostnames = await ResolveHostnamesAsync(
gatewayResource.Hostnames,
gatewayResource.Name,
cancellationToken).ConfigureAwait(false);

gateway.Spec.GatewayClassName = await ResolveExpressionAsync(gatewayResource.GatewayClassName, gatewayResource.Name, cancellationToken).ConfigureAwait(false);

Expand All @@ -1024,7 +1073,7 @@ private async Task BuildGatewayObjects(
{
var resolvedSecretName = await ResolveExpressionAsync(tls.SecretName, gatewayResource.Name, cancellationToken).ConfigureAwait(false);

if (gatewayResource.Hostnames.Count == 0)
if (resolvedHostnames.Count == 0)
{
// No hostnames specified — create an HTTPS listener without a hostname restriction.
// The hostname will be discovered from the Gateway's assigned address after deployment
Expand All @@ -1050,13 +1099,11 @@ private async Task BuildGatewayObjects(
}
else
{
foreach (var host in gatewayResource.Hostnames)
foreach (var resolvedHost in resolvedHostnames)
{
var listenerName = tlsListenerIndex == 0 ? "https" : $"https-{tlsListenerIndex}";
tlsListenerIndex++;

var resolvedHost = await ResolveExpressionAsync(host, gatewayResource.Name, cancellationToken).ConfigureAwait(false);

gateway.Spec.Listeners.Add(new GatewayListenerV1
{
Name = listenerName,
Expand Down Expand Up @@ -1098,6 +1145,10 @@ private async Task BuildGatewayObjects(
{
httpRoute.Spec.Hostnames.Add(hostGroup.Key);
}
else
{
httpRoute.Spec.Hostnames.AddRange(resolvedHostnames);
}

foreach (var route in hostGroup)
{
Expand Down
12 changes: 7 additions & 5 deletions src/Aspire.Hosting.Kubernetes/KubernetesGatewayExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,10 @@ public static IResourceBuilder<KubernetesGatewayResource> WithGatewayClass(
}

/// <summary>
/// Adds a path-based routing rule to the gateway. The rule matches all hosts and routes
/// traffic matching the specified path to the given endpoint's backing Kubernetes service.
/// This generates an <c>HTTPRoute</c> resource attached to the Gateway.
/// Adds a path-based routing rule to the gateway. The rule matches each hostname configured
/// with <see cref="WithHostname(IResourceBuilder{KubernetesGatewayResource}, string)"/>, or all
/// hosts when no hostname is configured, and routes matching traffic to the endpoint's backing
/// Kubernetes service. This generates an <c>HTTPRoute</c> resource attached to the Gateway.
/// </summary>
/// <param name="builder">The gateway resource builder.</param>
/// <param name="path">The URL path to match (e.g., <c>"/"</c> or <c>"/api"</c>). Must start with <c>/</c>.</param>
Expand Down Expand Up @@ -165,8 +166,9 @@ public static IResourceBuilder<KubernetesGatewayResource> WithRoute(

/// <summary>
/// Adds a hostname that this gateway's routes match. Multiple hostnames can be added by calling
/// this method repeatedly. Hostnames are used as <c>hostnames</c> in generated <c>HTTPRoute</c>
/// resources and as HTTPS listener hostnames when TLS is configured.
/// this method repeatedly. Routes without an explicit host apply to each configured hostname.
/// Hostnames are used as <c>hostnames</c> in generated <c>HTTPRoute</c> resources and as HTTPS
/// listener hostnames when TLS is configured.
/// </summary>
/// <param name="builder">The gateway resource builder.</param>
/// <param name="hostname">The hostname to match (e.g., <c>"api.example.com"</c>).</param>
Expand Down
12 changes: 9 additions & 3 deletions src/Aspire.Hosting.Kubernetes/KubernetesIngressExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ public static IResourceBuilder<KubernetesIngressResource> WithIngressClass(
}

/// <summary>
/// Adds a path-based rule to the ingress. The rule matches all hosts and forwards
/// traffic matching the specified path to the given endpoint's backing Kubernetes service.
/// Adds a path-based rule to the ingress. The rule matches each hostname configured with
/// <see cref="WithHostname(IResourceBuilder{KubernetesIngressResource}, string)"/>, or all hosts
/// when no hostname is configured, and forwards matching traffic to the endpoint's backing Kubernetes service.
/// </summary>
/// <param name="builder">The ingress resource builder.</param>
/// <param name="path">The URL path to match (e.g., <c>"/"</c> or <c>"/api"</c>). Must start with <c>/</c>.</param>
Expand Down Expand Up @@ -185,7 +186,8 @@ public static IResourceBuilder<KubernetesIngressResource> WithPath(

/// <summary>
/// Adds a hostname that this ingress matches. Multiple hostnames can be added by calling
/// this method repeatedly. If no hostnames are configured, the ingress matches all hosts.
/// this method repeatedly. Path rules without an explicit host apply to each configured
/// hostname. If no hostnames are configured, those rules match all hosts.
/// </summary>
/// <param name="builder">The ingress resource builder.</param>
/// <param name="hostname">The hostname to match (e.g., <c>"api.example.com"</c>).</param>
Expand Down Expand Up @@ -300,6 +302,10 @@ public static IResourceBuilder<KubernetesIngressResource> WithTls(
/// <param name="endpoint">The endpoint reference identifying the default backend service and port.</param>
/// <returns>A reference to the <see cref="IResourceBuilder{KubernetesIngressResource}"/> for chaining.</returns>
/// <ats-returns>The resource builder.</ats-returns>
/// <remarks>
/// Kubernetes default backends remain catch-all even when hostnames are configured. Use host-specific
/// path rules when traffic must be restricted by hostname.
/// </remarks>
[AspireExport]
public static IResourceBuilder<KubernetesIngressResource> WithDefaultBackend(
this IResourceBuilder<KubernetesIngressResource> builder,
Expand Down
Loading
Loading