diff --git a/.github/workflows/build-samples.yml b/.github/workflows/build-samples.yml index 97ab896..6793fe5 100644 --- a/.github/workflows/build-samples.yml +++ b/.github/workflows/build-samples.yml @@ -18,6 +18,7 @@ on: - "blazor/forms-validation-masterclass/**" - "blazor/ssr-interactive-islands/**" - "cloud-native/dockerize-aspnet-core-clean-images/**" + - "cloud-native/health-resilience-zero-downtime/**" - ".github/workflows/build-samples.yml" pull_request: @@ -36,6 +37,7 @@ on: - "blazor/forms-validation-masterclass/**" - "blazor/ssr-interactive-islands/**" - "cloud-native/dockerize-aspnet-core-clean-images/**" + - "cloud-native/health-resilience-zero-downtime/**" - ".github/workflows/build-samples.yml" workflow_dispatch: @@ -619,3 +621,38 @@ jobs: containerized-api-ci \ 2>/dev/null \ || true + + test-health-resilience-shutdown: + name: Test health, resilience, and shutdown sample + runs-on: ubuntu-latest + + permissions: + contents: read + + steps: + - name: Check out repository + uses: actions/checkout@v5 + + - name: Install .NET 10 SDK + uses: actions/setup-dotnet@v5 + with: + dotnet-version: "10.0.x" + + - name: Restore + run: > + dotnet restore + cloud-native/health-resilience-zero-downtime/ResilientOrdersMinimal.slnx + + - name: Build + run: > + dotnet build + cloud-native/health-resilience-zero-downtime/ResilientOrdersMinimal.slnx + --configuration Release + --no-restore + + - name: Test + run: > + dotnet test + cloud-native/health-resilience-zero-downtime/ResilientOrdersMinimal.slnx + --configuration Release + --no-build diff --git a/README.md b/README.md index 5ecec8b..8a7034c 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Each sample folder contains a focused implementation of one tutorial topic. The | [`blazor/forms-validation-masterclass`](blazor/forms-validation-masterclass/) | Focused .NET 10 Profile Settings form demonstrating manual EditContext management, DataAnnotations, FluentValidation, backend field-error mapping, accessible inputs, dirty state, and bUnit testing | [Blazor .NET 8 Forms & Validation: EditForm, FluentValidation & Server Error Handling](https://www.dotnet-guide.com/tutorials/blazor/forms-validation-masterclass/) | | [`blazor/ssr-interactive-islands`](blazor/ssr-interactive-islands/) | Focused .NET 10 catalog demonstrating static SSR, streaming review updates, a serializable render-mode boundary, an Interactive Server cart island, and component/integration testing | [Blazor SSR & Interactive Islands: Streaming Rendering, Auto Render Mode & Progressive Enhancement](https://www.dotnet-guide.com/tutorials/blazor/ssr-interactive-islands/) | | [`cloud-native/dockerize-aspnet-core-clean-images`](cloud-native/dockerize-aspnet-core-clean-images/) | Focused .NET 10 container sample demonstrating a multi-stage Dockerfile, locked restore, runtime-only image, non-root execution, port 8080, runtime configuration, health checks, hardened Compose settings, and CI smoke testing | [Dockerizing ASP.NET Core: Multi-Stage Builds, Clean Images & a Production-Ready Ship Workflow](https://www.dotnet-guide.com/tutorials/cloud-native/dockerize-aspnet-core-clean-images/) | +| [`cloud-native/health-resilience-zero-downtime`](cloud-native/health-resilience-zero-downtime/) | Focused .NET 10 Orders API demonstrating tagged liveness/readiness checks, structured health output, shutdown readiness draining, typed HttpClient retries, circuit breaking, attempt timeouts, and deterministic integration testing | [.NET 8 Cloud-Native: Health Probes, Polly Resilience & Zero-Downtime Kubernetes Deployments](https://www.dotnet-guide.com/tutorials/cloud-native/health-resilience-zero-downtime/) | ## Companion articles - [Common Microsoft.Extensions.AI mistakes](https://www.dotnet-guide.com/articles/dotnet-ai/microsoft-extensions-ai-common-mistakes/) @@ -203,26 +204,47 @@ tutorials/ | |-- BlazorCatalogIslands.Tests.csproj | `-- CatalogIslandTests.cs |-- cloud-native/ -| `-- dockerize-aspnet-core-clean-images/ -| |-- ContainerizedApiMinimal.slnx -| |-- Dockerfile -| |-- .dockerignore -| |-- compose.yaml +| |-- dockerize-aspnet-core-clean-images/ +| | |-- ContainerizedApiMinimal.slnx +| | |-- Dockerfile +| | |-- .dockerignore +| | |-- compose.yaml +| | |-- README.md +| | |-- src/ +| | | |-- ContainerizedApiMinimal/ +| | | | |-- ContainerizedApiMinimal.csproj +| | | | |-- Program.cs +| | | | |-- appsettings.json +| | | | `-- packages.lock.json +| | | `-- ContainerHealthProbe/ +| | | |-- ContainerHealthProbe.csproj +| | | |-- Program.cs +| | | `-- packages.lock.json +| | `-- tests/ +| | `-- ContainerizedApiMinimal.Tests/ +| | |-- ContainerizedApiMinimal.Tests.csproj +| | `-- ContainerizedApiTests.cs +| `-- health-resilience-zero-downtime/ | |-- README.md +| |-- ResilientOrdersMinimal.slnx | |-- src/ -| | |-- ContainerizedApiMinimal/ -| | | |-- ContainerizedApiMinimal.csproj -| | | |-- Program.cs -| | | |-- appsettings.json -| | | `-- packages.lock.json -| | `-- ContainerHealthProbe/ -| | |-- ContainerHealthProbe.csproj +| | `-- ResilientOrdersMinimal/ +| | |-- ResilientOrdersMinimal.csproj | | |-- Program.cs -| | `-- packages.lock.json +| | |-- Health/ +| | | |-- HealthResponseWriter.cs +| | | |-- TrafficReadinessHealthCheck.cs +| | | `-- TrafficReadinessState.cs +| | |-- Hosting/ +| | | `-- ShutdownReadinessService.cs +| | `-- Payments/ +| | |-- PaymentGatewayClient.cs +| | |-- PaymentSimulationState.cs +| | `-- SimulatedPaymentHandler.cs | `-- tests/ -| `-- ContainerizedApiMinimal.Tests/ -| |-- ContainerizedApiMinimal.Tests.csproj -| `-- ContainerizedApiTests.cs +| `-- ResilientOrdersMinimal.Tests/ +| |-- ResilientOrdersMinimal.Tests.csproj +| `-- HealthAndResilienceTests.cs |-- aspnet-core/ | |-- api-security-in-practice/ | | |-- ApiSecurityMinimal.slnx diff --git a/cloud-native/health-resilience-zero-downtime/README.md b/cloud-native/health-resilience-zero-downtime/README.md new file mode 100644 index 0000000..4b7c298 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/README.md @@ -0,0 +1,287 @@ +# ASP.NET Core Health, Resilience, and Shutdown Readiness + +A focused .NET 10 companion demonstrating tagged liveness and readiness checks, +structured health responses, readiness draining during shutdown, a typed +`HttpClient`, retries, a circuit breaker, an attempt timeout, and deterministic +integration tests. + +## Full tutorial + +[.NET 8 Cloud-Native: Health Probes, Polly Resilience & Zero-Downtime Kubernetes Deployments](https://www.dotnet-guide.com/tutorials/cloud-native/health-resilience-zero-downtime/) + +## Framework note + +The tutorial explains cloud-native ASP.NET Core patterns on .NET 8. + +This companion targets .NET 10 so it can use the repository's current SDK and +CI workflow. The tagged health-check, shutdown-readiness, typed-HttpClient, +retry, circuit-breaker, timeout, and integration-testing concepts demonstrated +here are the same core patterns. + +## What this sample demonstrates + +- `live` and `ready` health-check tags; +- `/health/live`; +- `/health/ready`; +- `/health`; +- structured JSON health responses; +- process liveness independent from traffic readiness; +- a hosted service that marks readiness unavailable during shutdown; +- `Microsoft.Extensions.Http.Resilience`; +- one custom resilience handler; +- retrying HTTP 503 responses; +- not retrying HTTP 400 responses; +- a circuit breaker; +- an attempt timeout; +- deterministic attempt counts; +- six integration tests. + +## Health semantics + +```text +Normal: + liveness -> Healthy + readiness -> Healthy + +Draining: + liveness -> Healthy + readiness -> Unhealthy +``` + +Liveness doesn't call an external dependency. + +Readiness represents whether this instance should receive new traffic. + +## Resilience pipeline + +```text +typed HttpClient + -> retry + -> circuit breaker + -> attempt timeout + -> simulated payment handler +``` + +The retry and circuit-breaker strategies handle HTTP 503 only. + +HTTP 400 is returned after one attempt. + +## Important POST retry warning + +The sample uses a POST-shaped authorization call because it makes the retry +behavior easy to understand. + +The handler is entirely in process and causes no external side effect. + +Do not blindly retry a real payment POST. Use an idempotency key or another +reviewed idempotency contract. + +## Deterministic dependency simulation + +`SimulatedPaymentHandler` doesn't use the network. + +It returns a requested number of temporary failures and then succeeds. + +This keeps the resilience tests: + +- fast; +- infrastructure-free; +- deterministic; +- safe for CI. + +It is not a production payment client or service-virtualization platform. + +## Prerequisite + +- .NET 10 SDK + +## Restore, build, and test + +```powershell +dotnet restore ` + .\ResilientOrdersMinimal.slnx + +dotnet build ` + .\ResilientOrdersMinimal.slnx ` + --configuration Release ` + --no-restore + +dotnet test ` + .\ResilientOrdersMinimal.slnx ` + --configuration Release ` + --no-build +``` + +## Run + +```powershell +dotnet run ` + --project .\src\ResilientOrdersMinimal\ResilientOrdersMinimal.csproj ` + --urls http://localhost:5156 +``` + +## Health endpoints + +```text +http://localhost:5156/health/live +http://localhost:5156/health/ready +http://localhost:5156/health +``` + +## Retry example + +```powershell +Invoke-WebRequest ` + -Method Post ` + -Uri "http://localhost:5156/api/orders/42/authorize?failuresBeforeSuccess=2&failureStatusCode=503" +``` + +Expected: + +```text +Succeeded: true +Attempts: 3 +CircuitOpen: false +``` + +## Non-retriable example + +Restart the app, then run: + +```powershell +Invoke-WebRequest ` + -Method Post ` + -SkipHttpErrorCheck ` + -Uri "http://localhost:5156/api/orders/43/authorize?failuresBeforeSuccess=10&failureStatusCode=400" +``` + +Expected: + +```text +HTTP 400 +Attempts: 1 +CircuitOpen: false +``` + +## Attempt-timeout example + +The handler records one attempt, waits for a delay that exceeds the configured +two-second attempt timeout, and returns HTTP 504 Gateway Timeout. + +Timeout exceptions are deliberately not retried in this sample. Only HTTP 503 +responses trigger the retry strategy. + +The delay exists solely for deterministic education and testing. + +Restart the app, then run: + +```powershell +Invoke-WebRequest ` + -Method Post ` + -SkipHttpErrorCheck ` + -Uri "http://localhost:5156/api/orders/45/authorize?failuresBeforeSuccess=0&failureStatusCode=503&delayMilliseconds=2500" +``` + +Expected: + +```text +HTTP 504 +Attempts: 1 +CircuitOpen: false +``` + +Do not treat this timeout handling as a production payment policy. + +## Circuit-breaker example + +Restart the app and run this twice immediately: + +```powershell +Invoke-WebRequest ` + -Method Post ` + -SkipHttpErrorCheck ` + -Uri "http://localhost:5156/api/orders/44/authorize?failuresBeforeSuccess=20&failureStatusCode=503" +``` + +First request: + +```text +Attempts: 3 +CircuitOpen: false +``` + +Second request: + +```text +Attempts: 0 +CircuitOpen: true +``` + +## Shutdown boundary + +`ShutdownReadinessService` changes traffic readiness when application shutdown +begins. + +It doesn't: + +- remove a Kubernetes endpoint; +- delay SIGTERM; +- wait for a load balancer; +- guarantee zero downtime. + +Those behaviors depend on the deployment platform and its probe, lifecycle, and +termination settings. + +## Project structure + +```text +ResilientOrdersMinimal.slnx +README.md +src/ +`-- ResilientOrdersMinimal/ + |-- ResilientOrdersMinimal.csproj + |-- Program.cs + |-- Health/ + | |-- HealthResponseWriter.cs + | |-- TrafficReadinessHealthCheck.cs + | `-- TrafficReadinessState.cs + |-- Hosting/ + | `-- ShutdownReadinessService.cs + `-- Payments/ + |-- PaymentGatewayClient.cs + |-- PaymentSimulationState.cs + `-- SimulatedPaymentHandler.cs +tests/ +`-- ResilientOrdersMinimal.Tests/ + |-- ResilientOrdersMinimal.Tests.csproj + `-- HealthAndResilienceTests.cs +``` + +## Deliberately omitted + +- databases; +- queues; +- migrations; +- background order processing; +- Docker; +- Kubernetes; +- OpenTelemetry; +- load testing; +- secrets; +- production rollout automation. + +These remain in the full tutorial or other focused samples. + +## Verification + +- Companion target framework: .NET 10 +- Tutorial framework: .NET 8 +- Resilience package: Microsoft.Extensions.Http.Resilience 10.8.0 +- External services required: none +- Database required: none +- Container runtime required: none +- Expected tests: 6 +- Last reviewed: 2026-08-03 + +This sample is educational and should be reviewed against the semantics of the +real downstream operation and deployment platform before production use. \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/ResilientOrdersMinimal.slnx b/cloud-native/health-resilience-zero-downtime/ResilientOrdersMinimal.slnx new file mode 100644 index 0000000..fe4deab --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/ResilientOrdersMinimal.slnx @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/HealthResponseWriter.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/HealthResponseWriter.cs new file mode 100644 index 0000000..02d4945 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/HealthResponseWriter.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ResilientOrdersMinimal.Health; + +public static class HealthResponseWriter +{ + public static Task WriteAsync( + HttpContext context, + HealthReport report) + { + context.Response.ContentType = + "application/json"; + + var response = + new + { + status = + report.Status.ToString(), + + durationMilliseconds = + report.TotalDuration + .TotalMilliseconds, + + checks = + report.Entries + .OrderBy( + entry => + entry.Key) + .Select( + entry => + new + { + name = + entry.Key, + + status = + entry.Value.Status + .ToString(), + + description = + entry.Value + .Description, + + durationMilliseconds = + entry.Value.Duration + .TotalMilliseconds + }) + }; + + return context.Response + .WriteAsJsonAsync( + response, + cancellationToken: + context.RequestAborted); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/TrafficReadinessHealthCheck.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/TrafficReadinessHealthCheck.cs new file mode 100644 index 0000000..2f5d3b1 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/TrafficReadinessHealthCheck.cs @@ -0,0 +1,28 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ResilientOrdersMinimal.Health; + +public sealed class TrafficReadinessHealthCheck( + TrafficReadinessState state) : + IHealthCheck +{ + public Task + CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = + default) + { + cancellationToken + .ThrowIfCancellationRequested(); + + HealthCheckResult result = + state.IsAcceptingTraffic + ? HealthCheckResult.Healthy( + "The API is accepting traffic.") + : HealthCheckResult.Unhealthy( + "The API is draining and must not receive new traffic."); + + return Task.FromResult( + result); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/TrafficReadinessState.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/TrafficReadinessState.cs new file mode 100644 index 0000000..0320e61 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Health/TrafficReadinessState.cs @@ -0,0 +1,19 @@ +namespace ResilientOrdersMinimal.Health; + +public sealed class TrafficReadinessState +{ + private int _acceptingTraffic = + 1; + + public bool IsAcceptingTraffic => + Volatile.Read( + ref _acceptingTraffic) + == 1; + + public void BeginDrain() + { + Interlocked.Exchange( + ref _acceptingTraffic, + 0); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Hosting/ShutdownReadinessService.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Hosting/ShutdownReadinessService.cs new file mode 100644 index 0000000..4a3bd94 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Hosting/ShutdownReadinessService.cs @@ -0,0 +1,45 @@ +using ResilientOrdersMinimal.Health; + +namespace ResilientOrdersMinimal.Hosting; + +public sealed class ShutdownReadinessService( + IHostApplicationLifetime lifetime, + TrafficReadinessState readiness, + ILogger logger) : + IHostedService, + IDisposable +{ + private CancellationTokenRegistration + _stoppingRegistration; + + public Task StartAsync( + CancellationToken cancellationToken) + { + _stoppingRegistration = + lifetime.ApplicationStopping + .Register( + () => + { + readiness.BeginDrain(); + + logger.LogInformation( + "Application shutdown started; readiness is now unavailable."); + }); + + return Task.CompletedTask; + } + + public Task StopAsync( + CancellationToken cancellationToken) + { + readiness.BeginDrain(); + + return Task.CompletedTask; + } + + public void Dispose() + { + _stoppingRegistration + .Dispose(); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/PaymentGatewayClient.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/PaymentGatewayClient.cs new file mode 100644 index 0000000..97144bf --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/PaymentGatewayClient.cs @@ -0,0 +1,132 @@ +using System.Globalization; +using System.Net; +using Polly.CircuitBreaker; +using Polly.Timeout; + +namespace ResilientOrdersMinimal.Payments; + +public sealed class PaymentGatewayClient( + HttpClient client, + PaymentSimulationState state) +{ + public async Task< + PaymentAuthorizationResult> + AuthorizeAsync( + int orderId, + int failuresBeforeSuccess, + HttpStatusCode failureStatus, + int delayMilliseconds, + CancellationToken cancellationToken) + { + string operationId = + Guid.NewGuid() + .ToString( + "N", + CultureInfo.InvariantCulture); + + using var request = + new HttpRequestMessage( + HttpMethod.Post, + $"/payments/orders/{orderId}/authorize"); + + request.Headers.Add( + "X-Demo-Operation-Id", + operationId); + + request.Headers.Add( + "X-Demo-Failures-Before-Success", + failuresBeforeSuccess + .ToString( + CultureInfo.InvariantCulture)); + + request.Headers.Add( + "X-Demo-Failure-Status", + ((int)failureStatus) + .ToString( + CultureInfo.InvariantCulture)); + + request.Headers.Add( + "X-Demo-Delay-Milliseconds", + delayMilliseconds + .ToString( + CultureInfo.InvariantCulture)); + + try + { + using HttpResponseMessage response = + await client.SendAsync( + request, + cancellationToken); + + return new PaymentAuthorizationResult( + OrderId: + orderId, + + Succeeded: + response.IsSuccessStatusCode, + + Attempts: + state.GetAttempts( + operationId), + + CircuitOpen: + false, + + StatusCode: + (int)response.StatusCode); + } + catch (BrokenCircuitException) + { + return new PaymentAuthorizationResult( + OrderId: + orderId, + + Succeeded: + false, + + Attempts: + state.GetAttempts( + operationId), + + CircuitOpen: + true, + + StatusCode: + StatusCodes + .Status503ServiceUnavailable); + } + catch (TimeoutRejectedException) + { + return new PaymentAuthorizationResult( + OrderId: + orderId, + + Succeeded: + false, + + Attempts: + state.GetAttempts( + operationId), + + CircuitOpen: + false, + + StatusCode: + StatusCodes + .Status504GatewayTimeout); + } + finally + { + state.Remove( + operationId); + } + } +} + +public sealed record + PaymentAuthorizationResult( + int OrderId, + bool Succeeded, + int Attempts, + bool CircuitOpen, + int StatusCode); \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/PaymentSimulationState.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/PaymentSimulationState.cs new file mode 100644 index 0000000..6d813f3 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/PaymentSimulationState.cs @@ -0,0 +1,41 @@ +using System.Collections.Concurrent; + +namespace ResilientOrdersMinimal.Payments; + +public sealed class PaymentSimulationState +{ + private readonly + ConcurrentDictionary< + string, + int> _attempts = + new( + StringComparer.Ordinal); + + public int RecordAttempt( + string operationId) => + _attempts.AddOrUpdate( + operationId, + addValue: + 1, + updateValueFactory: + static ( + key, + current) => + current + 1); + + public int GetAttempts( + string operationId) => + _attempts.TryGetValue( + operationId, + out int attempts) + ? attempts + : 0; + + public void Remove( + string operationId) + { + _attempts.TryRemove( + operationId, + out _); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/SimulatedPaymentHandler.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/SimulatedPaymentHandler.cs new file mode 100644 index 0000000..02cd547 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Payments/SimulatedPaymentHandler.cs @@ -0,0 +1,116 @@ +using System.Globalization; +using System.Net; +using System.Net.Http.Json; + +namespace ResilientOrdersMinimal.Payments; + +public sealed class SimulatedPaymentHandler( + PaymentSimulationState state) : + HttpMessageHandler +{ + protected override async Task< + HttpResponseMessage> SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + cancellationToken + .ThrowIfCancellationRequested(); + + string operationId = + ReadRequiredHeader( + request, + "X-Demo-Operation-Id"); + + int failuresBeforeSuccess = + int.Parse( + ReadRequiredHeader( + request, + "X-Demo-Failures-Before-Success"), + CultureInfo.InvariantCulture); + + int failureStatusCode = + int.Parse( + ReadRequiredHeader( + request, + "X-Demo-Failure-Status"), + CultureInfo.InvariantCulture); + + int delayMilliseconds = + int.Parse( + ReadRequiredHeader( + request, + "X-Demo-Delay-Milliseconds"), + CultureInfo.InvariantCulture); + + int attempt = + state.RecordAttempt( + operationId); + + if (delayMilliseconds > 0) + { + await Task.Delay( + delayMilliseconds, + cancellationToken); + } + + bool shouldFail = + attempt + <= failuresBeforeSuccess; + + HttpStatusCode statusCode = + shouldFail + ? (HttpStatusCode) + failureStatusCode + : HttpStatusCode.OK; + + var response = + new HttpResponseMessage( + statusCode) + { + RequestMessage = + request, + + Content = + JsonContent.Create( + new + { + authorized = + !shouldFail, + + attempt + }) + }; + + response.Headers.Add( + "X-Demo-Attempt", + attempt.ToString( + CultureInfo.InvariantCulture)); + + return response; + } + + private static string + ReadRequiredHeader( + HttpRequestMessage request, + string name) + { + if (request.Headers + .TryGetValues( + name, + out IEnumerable< + string>? values)) + { + string? value = + values.SingleOrDefault(); + + if (!string.IsNullOrWhiteSpace( + value)) + { + return value; + } + } + + throw new InvalidOperationException( + $"Required demonstration header '{name}' is missing."); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Program.cs b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Program.cs new file mode 100644 index 0000000..f160f85 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/Program.cs @@ -0,0 +1,295 @@ +using System.Net; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Http.Resilience; +using Polly; +using ResilientOrdersMinimal.Health; +using ResilientOrdersMinimal.Hosting; +using ResilientOrdersMinimal.Payments; + +var builder = + WebApplication.CreateBuilder(args); + +builder.Host.ConfigureHostOptions( + options => + { + options.ShutdownTimeout = + TimeSpan.FromSeconds( + 30); + }); + +builder.Services.AddSingleton< + TrafficReadinessState>(); + +builder.Services.AddSingleton< + PaymentSimulationState>(); + +builder.Services.AddSingleton< + ShutdownReadinessService>(); + +builder.Services.AddHostedService( + services => + services.GetRequiredService< + ShutdownReadinessService>()); + +builder.Services + .AddHealthChecks() + .AddCheck( + "self", + () => + HealthCheckResult.Healthy( + "The process is responding."), + tags: + [ + "live" + ]) + .AddCheck< + TrafficReadinessHealthCheck>( + "traffic-readiness", + tags: + [ + "ready" + ]); + +builder.Services + .AddHttpClient< + PaymentGatewayClient>( + client => + { + client.BaseAddress = + new Uri( + "https://payment.example.invalid"); + }) + .ConfigurePrimaryHttpMessageHandler( + services => + new SimulatedPaymentHandler( + services + .GetRequiredService< + PaymentSimulationState>())) + .AddResilienceHandler( + "payment-pipeline", + resilience => + { + resilience + .AddRetry( + new HttpRetryStrategyOptions + { + MaxRetryAttempts = + 2, + + Delay = + TimeSpan + .FromMilliseconds( + 20), + + BackoffType = + DelayBackoffType + .Exponential, + + UseJitter = + true, + + ShouldHandle = + static arguments => + ValueTask + .FromResult( + arguments + .Outcome + .Result? + .StatusCode + == + HttpStatusCode + .ServiceUnavailable) + }) + .AddCircuitBreaker( + new HttpCircuitBreakerStrategyOptions + { + FailureRatio = + 1.0, + + MinimumThroughput = + 3, + + SamplingDuration = + TimeSpan + .FromSeconds( + 30), + + BreakDuration = + TimeSpan + .FromSeconds( + 10), + + ShouldHandle = + static arguments => + ValueTask + .FromResult( + arguments + .Outcome + .Result? + .StatusCode + == + HttpStatusCode + .ServiceUnavailable) + }) + .AddTimeout( + TimeSpan.FromSeconds( + 2)); + }); + +var app = + builder.Build(); + +app.MapGet( + "/", + () => + TypedResults.Ok( + new + { + name = + "ResilientOrdersMinimal", + + endpoints = + new[] + { + "GET /health/live", + "GET /health/ready", + "GET /health", + "POST /api/orders/{id}/authorize" + }, + + note = + "The payment dependency is simulated in process for deterministic resilience testing." + })); + +app.MapPost( + "/api/orders/{id:int:min(1)}/authorize", + AuthorizeOrderAsync); + +app.MapHealthChecks( + "/health/live", + new HealthCheckOptions + { + Predicate = + registration => + registration.Tags + .Contains( + "live"), + + ResponseWriter = + HealthResponseWriter + .WriteAsync + }); + +app.MapHealthChecks( + "/health/ready", + new HealthCheckOptions + { + Predicate = + registration => + registration.Tags + .Contains( + "ready"), + + ResponseWriter = + HealthResponseWriter + .WriteAsync + }); + +app.MapHealthChecks( + "/health", + new HealthCheckOptions + { + ResponseWriter = + HealthResponseWriter + .WriteAsync + }); + +app.Run(); + +static async Task + AuthorizeOrderAsync( + int id, + int failuresBeforeSuccess, + int failureStatusCode, + int delayMilliseconds, + PaymentGatewayClient client, + CancellationToken cancellationToken) +{ + if (failuresBeforeSuccess + is < 0 + or > 20) + { + return Results.ValidationProblem( + new Dictionary< + string, + string[]> + { + [nameof( + failuresBeforeSuccess)] = + [ + "Use a value from 0 through 20." + ] + }); + } + + if (failureStatusCode + is not 400 + and not 503) + { + return Results.ValidationProblem( + new Dictionary< + string, + string[]> + { + [nameof( + failureStatusCode)] = + [ + "Use 400 for a non-retriable failure or 503 for a retriable failure." + ] + }); + } + + if (delayMilliseconds + is < 0 + or > 5000) + { + return Results.ValidationProblem( + new Dictionary< + string, + string[]> + { + [nameof( + delayMilliseconds)] = + [ + "Use a value from 0 through 5000." + ] + }); + } + + PaymentAuthorizationResult result = + await client.AuthorizeAsync( + id, + failuresBeforeSuccess, + (HttpStatusCode) + failureStatusCode, + delayMilliseconds, + cancellationToken); + + int responseStatus = + result.Succeeded + ? StatusCodes.Status200OK + : result.CircuitOpen + ? StatusCodes + .Status503ServiceUnavailable + : result.StatusCode; + + return Results.Json( + result, + statusCode: + responseStatus); +} + +public partial class Program +{ +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/ResilientOrdersMinimal.csproj b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/ResilientOrdersMinimal.csproj new file mode 100644 index 0000000..0f40f4f --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/src/ResilientOrdersMinimal/ResilientOrdersMinimal.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + enable + enable + + + + + + + \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/tests/ResilientOrdersMinimal.Tests/HealthAndResilienceTests.cs b/cloud-native/health-resilience-zero-downtime/tests/ResilientOrdersMinimal.Tests/HealthAndResilienceTests.cs new file mode 100644 index 0000000..c3dc2f5 --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/tests/ResilientOrdersMinimal.Tests/HealthAndResilienceTests.cs @@ -0,0 +1,599 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ResilientOrdersMinimal.Hosting; +using ResilientOrdersMinimal.Payments; + +namespace ResilientOrdersMinimal.Tests; + +public sealed class + HealthAndResilienceTests +{ + private static async Task< + JsonDocument> + ParseHealthBodyAsync( + HttpResponseMessage response, + CancellationToken ct) + { + string body = + await response.Content + .ReadAsStringAsync( + ct); + + return JsonDocument.Parse( + body); + } + + private static bool + CheckExists( + JsonElement checks, + string name) + { + return checks + .EnumerateArray() + .Any( + check => + check.GetProperty( + "name") + .GetString() + == name); + } + + [Fact] + public async Task + Root_describes_the_sample() + { + using var factory = + new WebApplicationFactory< + Program>(); + + using HttpClient client = + factory.CreateClient(); + + CancellationToken ct = + TestContext.Current + .CancellationToken; + + HttpResponseMessage response = + await client.GetAsync( + "/", + ct); + + string body = + await response.Content + .ReadAsStringAsync( + ct); + + Assert.Equal( + HttpStatusCode.OK, + response.StatusCode); + + Assert.Contains( + "ResilientOrdersMinimal", + body, + StringComparison.Ordinal); + + Assert.Contains( + "/health/ready", + body, + StringComparison.Ordinal); + } + + [Fact] + public async Task + Health_endpoints_are_healthy_and_structured() + { + using var factory = + new WebApplicationFactory< + Program>(); + + using HttpClient client = + factory.CreateClient(); + + CancellationToken ct = + TestContext.Current + .CancellationToken; + + HttpResponseMessage liveResponse = + await client.GetAsync( + "/health/live", + ct); + + HttpResponseMessage readyResponse = + await client.GetAsync( + "/health/ready", + ct); + + HttpResponseMessage combinedResponse = + await client.GetAsync( + "/health", + ct); + + Assert.Equal( + HttpStatusCode.OK, + liveResponse.StatusCode); + + Assert.Equal( + HttpStatusCode.OK, + readyResponse.StatusCode); + + Assert.Equal( + HttpStatusCode.OK, + combinedResponse.StatusCode); + + Assert.Equal( + "application/json", + readyResponse.Content.Headers + .ContentType? + .MediaType); + + // -- /health/live: exactly one check, name is self + using JsonDocument liveBody = + await ParseHealthBodyAsync( + liveResponse, + ct); + + Assert.Equal( + "Healthy", + liveBody.RootElement + .GetProperty("status") + .GetString()); + + JsonElement liveChecks = + liveBody.RootElement + .GetProperty("checks"); + + Assert.Equal( + 1, + liveChecks.GetArrayLength()); + + Assert.True( + CheckExists( + liveChecks, + "self")); + + Assert.False( + CheckExists( + liveChecks, + "traffic-readiness")); + + // -- /health/ready: exactly one check, name is traffic-readiness + using JsonDocument readyBody = + await ParseHealthBodyAsync( + readyResponse, + ct); + + Assert.Equal( + "Healthy", + readyBody.RootElement + .GetProperty("status") + .GetString()); + + JsonElement readyChecks = + readyBody.RootElement + .GetProperty("checks"); + + Assert.Equal( + 1, + readyChecks.GetArrayLength()); + + Assert.True( + CheckExists( + readyChecks, + "traffic-readiness")); + + Assert.False( + CheckExists( + readyChecks, + "self")); + + // -- /health: exactly two checks, both present + using JsonDocument combinedBody = + await ParseHealthBodyAsync( + combinedResponse, + ct); + + Assert.Equal( + "Healthy", + combinedBody.RootElement + .GetProperty("status") + .GetString()); + + JsonElement combinedChecks = + combinedBody.RootElement + .GetProperty("checks"); + + Assert.Equal( + 2, + combinedChecks.GetArrayLength()); + + Assert.True( + CheckExists( + combinedChecks, + "self")); + + Assert.True( + CheckExists( + combinedChecks, + "traffic-readiness")); + } + + [Fact] + public async Task + Shutdown_service_drains_readiness_but_not_liveness() + { + using var factory = + new WebApplicationFactory< + Program>(); + + using HttpClient client = + factory.CreateClient(); + + CancellationToken ct = + TestContext.Current + .CancellationToken; + + // Confirm the concrete service resolved from DI is the same + // instance registered as the hosted service. + ShutdownReadinessService + concreteService = + factory.Services + .GetRequiredService< + ShutdownReadinessService>(); + + IEnumerable + hostedServices = + factory.Services + .GetRequiredService< + IEnumerable< + IHostedService>>(); + + IHostedService + shutdownHostedService = + hostedServices + .OfType< + ShutdownReadinessService>() + .Single(); + + Assert.Same( + concreteService, + shutdownHostedService); + + await concreteService.StopAsync( + ct); + + // -- /health/live: 200, self remains Healthy + HttpResponseMessage liveResponse = + await client.GetAsync( + "/health/live", + ct); + + Assert.Equal( + HttpStatusCode.OK, + liveResponse.StatusCode); + + using JsonDocument liveBody = + await ParseHealthBodyAsync( + liveResponse, + ct); + + Assert.Equal( + "Healthy", + liveBody.RootElement + .GetProperty("status") + .GetString()); + + JsonElement liveChecks = + liveBody.RootElement + .GetProperty("checks"); + + Assert.True( + CheckExists( + liveChecks, + "self")); + + // -- /health/ready: 503, overall Unhealthy, traffic-readiness Unhealthy + HttpResponseMessage readyResponse = + await client.GetAsync( + "/health/ready", + ct); + + Assert.Equal( + HttpStatusCode + .ServiceUnavailable, + readyResponse.StatusCode); + + using JsonDocument readyBody = + await ParseHealthBodyAsync( + readyResponse, + ct); + + Assert.Equal( + "Unhealthy", + readyBody.RootElement + .GetProperty("status") + .GetString()); + + JsonElement readyChecks = + readyBody.RootElement + .GetProperty("checks"); + + Assert.True( + CheckExists( + readyChecks, + "traffic-readiness")); + + Assert.Equal( + "Unhealthy", + readyChecks + .EnumerateArray() + .First( + c => + c.GetProperty("name") + .GetString() + == "traffic-readiness") + .GetProperty("status") + .GetString()); + + // -- /health: 503, overall Unhealthy, self Healthy, traffic-readiness Unhealthy + HttpResponseMessage combinedResponse = + await client.GetAsync( + "/health", + ct); + + Assert.Equal( + HttpStatusCode + .ServiceUnavailable, + combinedResponse.StatusCode); + + using JsonDocument combinedBody = + await ParseHealthBodyAsync( + combinedResponse, + ct); + + Assert.Equal( + "Unhealthy", + combinedBody.RootElement + .GetProperty("status") + .GetString()); + + JsonElement combinedChecks = + combinedBody.RootElement + .GetProperty("checks"); + + Assert.Equal( + 2, + combinedChecks.GetArrayLength()); + + Assert.Equal( + "Healthy", + combinedChecks + .EnumerateArray() + .First( + c => + c.GetProperty("name") + .GetString() + == "self") + .GetProperty("status") + .GetString()); + + Assert.Equal( + "Unhealthy", + combinedChecks + .EnumerateArray() + .First( + c => + c.GetProperty("name") + .GetString() + == "traffic-readiness") + .GetProperty("status") + .GetString()); + } + + [Fact] + public async Task + Temporary_service_unavailability_is_retried() + { + using var factory = + new WebApplicationFactory< + Program>(); + + using HttpClient client = + factory.CreateClient(); + + CancellationToken ct = + TestContext.Current + .CancellationToken; + + HttpResponseMessage response = + await client.PostAsync( + "/api/orders/42/authorize?failuresBeforeSuccess=2&failureStatusCode=503&delayMilliseconds=0", + content: + null, + ct); + + PaymentAuthorizationResult? result = + await response.Content + .ReadFromJsonAsync< + PaymentAuthorizationResult>( + cancellationToken: + ct); + + Assert.Equal( + HttpStatusCode.OK, + response.StatusCode); + + Assert.NotNull( + result); + + Assert.True( + result.Succeeded); + + Assert.False( + result.CircuitOpen); + + Assert.Equal( + 3, + result.Attempts); + } + + [Fact] + public async Task + Bad_request_and_timeout_are_not_retried() + { + using var factory = + new WebApplicationFactory< + Program>(); + + using HttpClient client = + factory.CreateClient(); + + CancellationToken ct = + TestContext.Current + .CancellationToken; + + // -- HTTP 400: non-retriable, one attempt + HttpResponseMessage badRequestResponse = + await client.PostAsync( + "/api/orders/43/authorize?failuresBeforeSuccess=10&failureStatusCode=400&delayMilliseconds=0", + content: + null, + ct); + + PaymentAuthorizationResult? badRequestResult = + await badRequestResponse.Content + .ReadFromJsonAsync< + PaymentAuthorizationResult>( + cancellationToken: + ct); + + Assert.Equal( + HttpStatusCode.BadRequest, + badRequestResponse.StatusCode); + + Assert.NotNull( + badRequestResult); + + Assert.False( + badRequestResult.Succeeded); + + Assert.False( + badRequestResult.CircuitOpen); + + Assert.Equal( + 1, + badRequestResult.Attempts); + + // -- Attempt timeout: delay exceeds 2-second timeout, one attempt, 504 + HttpResponseMessage timeoutResponse = + await client.PostAsync( + "/api/orders/45/authorize?failuresBeforeSuccess=0&failureStatusCode=503&delayMilliseconds=2500", + content: + null, + ct); + + PaymentAuthorizationResult? timeoutResult = + await timeoutResponse.Content + .ReadFromJsonAsync< + PaymentAuthorizationResult>( + cancellationToken: + ct); + + Assert.Equal( + HttpStatusCode.GatewayTimeout, + timeoutResponse.StatusCode); + + Assert.NotNull( + timeoutResult); + + Assert.False( + timeoutResult.Succeeded); + + Assert.False( + timeoutResult.CircuitOpen); + + Assert.Equal( + 1, + timeoutResult.Attempts); + } + + [Fact] + public async Task + Repeated_service_unavailability_opens_the_circuit() + { + using var factory = + new WebApplicationFactory< + Program>(); + + using HttpClient client = + factory.CreateClient(); + + CancellationToken ct = + TestContext.Current + .CancellationToken; + + string path = + "/api/orders/44/authorize?failuresBeforeSuccess=20&failureStatusCode=503&delayMilliseconds=0"; + + HttpResponseMessage firstResponse = + await client.PostAsync( + path, + content: + null, + ct); + + PaymentAuthorizationResult? first = + await firstResponse.Content + .ReadFromJsonAsync< + PaymentAuthorizationResult>( + cancellationToken: + ct); + + HttpResponseMessage secondResponse = + await client.PostAsync( + path, + content: + null, + ct); + + PaymentAuthorizationResult? second = + await secondResponse.Content + .ReadFromJsonAsync< + PaymentAuthorizationResult>( + cancellationToken: + ct); + + Assert.Equal( + HttpStatusCode + .ServiceUnavailable, + firstResponse.StatusCode); + + Assert.NotNull( + first); + + Assert.Equal( + 3, + first.Attempts); + + Assert.False( + first.CircuitOpen); + + Assert.Equal( + HttpStatusCode + .ServiceUnavailable, + secondResponse.StatusCode); + + Assert.NotNull( + second); + + Assert.True( + second.CircuitOpen); + + Assert.Equal( + 0, + second.Attempts); + } +} \ No newline at end of file diff --git a/cloud-native/health-resilience-zero-downtime/tests/ResilientOrdersMinimal.Tests/ResilientOrdersMinimal.Tests.csproj b/cloud-native/health-resilience-zero-downtime/tests/ResilientOrdersMinimal.Tests/ResilientOrdersMinimal.Tests.csproj new file mode 100644 index 0000000..3672b5c --- /dev/null +++ b/cloud-native/health-resilience-zero-downtime/tests/ResilientOrdersMinimal.Tests/ResilientOrdersMinimal.Tests.csproj @@ -0,0 +1,49 @@ + + + + net10.0 + enable + enable + false + true + Exe + + + + + + + + + + + all + + runtime; + build; + native; + contentfiles; + analyzers; + buildtransitive + + + + + + + + + + + + + \ No newline at end of file