Prove automatic Kubernetes worker failure recovery

deploy/kubernetes/kind-cluster.yaml

@@ -6,6 +6,14 @@ nodes:
   - role: control-plane
     image: kindest/node:v1.34.3@sha256:08497ee19eace7b4b5348db5c6a1591d7752b164530a36f855cb0f2bdcbadd48
     kubeadmConfigPatches:
+      - |
+        # kind 0.31.0 uses kubeadm v1beta3 for Kubernetes 1.34.x.
+        apiVersion: kubeadm.k8s.io/v1beta3
+        kind: ClusterConfiguration
+        controllerManager:
+          extraArgs:
+            node-monitor-period: 2s
+            node-monitor-grace-period: 20s
       - |
         apiVersion: kubeproxy.config.k8s.io/v1alpha1
         kind: KubeProxyConfiguration

deploy/kubernetes/qualification.yaml

@@ -61,6 +61,15 @@ spec:
       enableServiceLinks: false
       nodeSelector:
         loadbalancerpro.io/qualification-worker: "true"
+      tolerations:
+        - key: node.kubernetes.io/not-ready
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: 10
+        - key: node.kubernetes.io/unreachable
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: 10
       topologySpreadConstraints:
         - maxSkew: 1
           minDomains: 2
@@ -140,6 +149,15 @@ spec:
       enableServiceLinks: false
       nodeSelector:
         loadbalancerpro.io/qualification-worker: "true"
+      tolerations:
+        - key: node.kubernetes.io/not-ready
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: 10
+        - key: node.kubernetes.io/unreachable
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: 10
       topologySpreadConstraints:
         - maxSkew: 1
           minDomains: 2
@@ -230,6 +248,15 @@ spec:
       terminationGracePeriodSeconds: 45
       nodeSelector:
         loadbalancerpro.io/qualification-worker: "true"
+      tolerations:
+        - key: node.kubernetes.io/not-ready
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: 10
+        - key: node.kubernetes.io/unreachable
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: 10
       topologySpreadConstraints:
         - maxSkew: 1
           minDomains: 2

docs/DEPLOYMENT.md

@@ -144,10 +144,16 @@ after stale pooled connections have been discarded. The lane then proves two-zon
 Service distribution, planned worker removal, and
 operator-remediated no-drain worker loss and recovery. The abrupt-loss exercise forcibly stops the kind worker,
 confirms its container is down, applies the out-of-service `NoExecute` taint, and force-removes the three exact stateless
-qualification pods from the API. The disposable cluster also pins immediate EndpointSlice-triggered iptables updates
-and a one-second kube-proxy cleanup sync; operators must review the equivalent setting or managed-ingress
-behavior for their environment. The candidate preserves the baseline application layers, so this does not prove
-application-layer release compatibility or automatic deployment failure detection. The reviewed staging runner
+qualification pods from the API. A separate no-drain kill then permits no post-failure Kubernetes mutation: the pinned
+controller-manager must detect the lost node, EndpointSlices must withdraw its endpoints, and the bounded `NoExecute`
+tolerations must initiate eviction for its three pods. The evidence accepts API removal or a deletion timestamp because
+an unavailable kubelet can leave the old objects terminating until it returns; fresh identities must still recover across
+both zones. The disposable cluster pins a
+twenty-second node-monitor grace period, ten-second unreachable tolerations, immediate EndpointSlice-triggered iptables
+updates, and a one-second kube-proxy cleanup sync; operators must review equivalent managed-control-plane and ingress
+behavior instead of copying those lab timings. Its controller-detected transition has an 80% success floor that includes
+the grace interval, followed by a 95% degraded floor and 99.9% recovered floor. The candidate preserves the baseline application layers, so this does not
+prove application-layer release compatibility or deployment-equivalent infrastructure failure timing. The reviewed staging runner
 separately validates the external target's digest, replicas, zones, resources,
 configuration, ingress, metrics, drain, and transitions; local proof does not establish registry integrity, deployment
 capacity, external ingress behavior, or production readiness.

docs/LOAD_BALANCER_BUILD_OUT.md

@@ -175,17 +175,23 @@ and second replica must recover inside the bound, and both recovered replicas an
 then forcibly stops that recovered worker without a drain, confirms the container is down, and applies the documented
 out-of-service `NoExecute` remediation. It force-removes the three exact stateless workload pods from the API, bounds
 endpoint withdrawal, proves degraded traffic, rejects the failed pod identity after recovery, and requires both
-recovered replicas and backends to serve new traffic. The disposable cluster pins iptables-mode kube-proxy
-to immediate EndpointSlice-triggered updates and a one-second cleanup sync; deployment environments must review the
-equivalent Service/ingress failure-detection and reconciliation behavior. Its abrupt transition and degraded phases
-bound stale conntrack impact at 90% and 95% success with 5.5-second p99 ceilings; recovered traffic must return to the
-normal 99.9% success and 1.5-second p99 objectives. The ten-second endpoint drain exceeds the five-second qualification
+recovered replicas and backends to serve new traffic. A second forced stop permits no post-failure Kubernetes mutation:
+the test-configured controller-manager must detect the lost node, EndpointSlices must withdraw its endpoints, the bounded
+`NoExecute` tolerations must initiate eviction for its three pods, and fresh pod identities must recover across both zones.
+Eviction evidence accepts API removal or a deletion timestamp because an unavailable kubelet can leave the old objects
+terminating until the node returns. The disposable
+cluster pins a twenty-second node-monitor grace period, ten-second unreachable tolerations, iptables-mode kube-proxy with
+immediate EndpointSlice-triggered updates, and a one-second cleanup sync; deployment environments must review equivalent
+managed-control-plane and Service/ingress reconciliation behavior. Its abrupt transition and degraded phases
+bound stale conntrack impact at 90% and 95% success with 5.5-second p99 ceilings. The controller-detected transition has
+an 80% floor that includes the configured node-monitor grace interval, followed by a 95% degraded floor; recovered
+traffic must return to the normal 99.9% success and 1.5-second p99 objectives. The ten-second endpoint drain exceeds the five-second qualification
 client timeout, and the 45-second termination grace contains the 30-second application shutdown bound. The candidate
 has a distinct local image
 content ID but preserves the baseline application layers, so it proves Kubernetes transition and rollback mechanics
 rather than compatibility between application releases. The reviewed deployment ingress, deployment-equivalent
 resources, registry
-digest transition, and automatic infrastructure-failure detection remain staging gates.
+digest transition, and deployment-equivalent infrastructure-failure timing remain staging gates.
 
 ### 4. Stage The Rollout And Rollback
 

scripts/bench/README.md

@@ -141,10 +141,17 @@ tests the one-replica degraded service, requires both recovered replicas and bac
 forcibly stops that recovered worker without a drain. After confirming the worker container is down, it applies
 Kubernetes' out-of-service `NoExecute` remediation and force-removes the three exact stateless workload pods from the
 API, bounds endpoint withdrawal, proves degraded traffic, and requires fresh pod identity, two-zone placement, and traffic
-distribution after recovery. The lab cluster pins iptables kube-proxy to immediate EndpointSlice-triggered updates and a
-one-second cleanup sync so the Service failover objective is executable and recorded. The abrupt transition and degraded
-windows retain bounded 90% and 95% success floors with 5.5-second p99 ceilings for stale conntrack paths; recovered
-traffic returns to the normal 99.9% success and 1.5-second p99 objectives. Rollouts drain endpoints for ten seconds,
+distribution after recovery. It then repeats a no-drain worker kill without any post-failure Kubernetes mutation. The
+controller-manager must detect the lost node, EndpointSlices must withdraw its endpoints, bounded `NoExecute` tolerations
+must initiate eviction for the three dead-node pods, and recovery must use fresh pod identities across both zones. The
+evidence accepts either API removal or a deletion timestamp because an unavailable kubelet can leave evicted pod objects
+terminating until the node returns. The lab cluster pins a
+twenty-second node-monitor grace period, ten-second unreachable tolerations, iptables kube-proxy with immediate
+EndpointSlice-triggered updates, and a one-second cleanup sync so the automatic Service failover objective is executable
+and recorded. Both abrupt transition and degraded windows retain bounded 90% and 95% success floors with 5.5-second p99
+ceilings for stale conntrack paths. The controller-detected transition has a separate 80% floor to include the configured
+node-monitor grace interval, while its degraded phase retains 95%; recovered traffic returns to the normal 99.9% success
+and 1.5-second p99 objectives. Rollouts drain endpoints for ten seconds,
 longer than the five-second qualification client timeout, while the 45-second grace period contains the application's
 30-second graceful-shutdown bound:
 
@@ -162,8 +169,8 @@ prove dynamic Secret reload or an external secret manager, and it does not deplo
 external certificate authority, or client trust-distribution system. Because the local candidate
 changes immutable proof metadata but not application layers, it does not prove application-layer release compatibility,
 registry integrity,
-deployment capacity, external ingress behavior, automatic infrastructure-failure detection, or an authorized staging
-environment.
+deployment capacity, external ingress behavior, deployment-equivalent infrastructure-failure timing, or an authorized
+staging environment.
 
 ## Local capacity staircase
 

scripts/bench/kubernetes-topology-contract-test.sh

@@ -14,7 +14,7 @@ trap cleanup EXIT
 
 bash -n "$runner"
 bash "$runner" --mode validate --profile "$profile" > "$work_dir/valid.log"
-grep -Fq 'service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery' \
+grep -Fq 'service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery automatic-node-detection automatic-endpoint-withdrawal automatic-pod-eviction automatic-recovery' \
     "$work_dir/valid.log"
 
 assert_rejected() {
@@ -33,12 +33,15 @@ assert_rejected legacy-schema '.schemaVersion = 1'
 assert_rejected older-schema '.schemaVersion = 3'
 assert_rejected older-schema-four '.schemaVersion = 4'
 assert_rejected previous-schema '.schemaVersion = 5'
+assert_rejected previous-schema-six '.schemaVersion = 6'
 assert_rejected skewed-kubectl '.cluster.kubectlVersion = "v1.32.2"'
 assert_rejected mutable-node-image '.cluster.nodeImageDigest = "sha256:" + ("f" * 64)'
 assert_rejected one-worker '.cluster.workers = 1'
 assert_rejected one-zone '.cluster.zones = 1'
 assert_rejected external-namespace '.cluster.namespace = "production"'
 assert_rejected public-port '.cluster.hostPort = 443'
+assert_rejected slow-node-monitor '.cluster.nodeMonitorGracePeriodSeconds = 50'
+assert_rejected slow-unreachable-toleration '.cluster.unreachableTolerationSeconds = 300'
 assert_rejected sticky-connection-mode '.workload.connectionMode = "keep-alive"'
 assert_rejected low-rate '.workload.ratePerSecond = 1'
 assert_rejected short-rollout '.workload.rolloutSeconds = 10'
@@ -61,6 +64,10 @@ assert_rejected short-abrupt-transition '.workload.abruptTransitionSeconds = 5'
 assert_rejected short-abrupt-degraded '.workload.abruptDegradedSeconds = 1'
 assert_rejected short-abrupt-recovered '.workload.abruptRecoveredSeconds = 1'
 assert_rejected abrupt-window-too-short '.workload.abruptTransitionSeconds = .objectives.maximumAbruptEndpointWithdrawalSeconds'
+assert_rejected short-automatic-transition '.workload.automaticTransitionSeconds = 10'
+assert_rejected short-automatic-degraded '.workload.automaticDegradedSeconds = 1'
+assert_rejected short-automatic-recovered '.workload.automaticRecoveredSeconds = 1'
+assert_rejected automatic-window-too-short '.workload.automaticTransitionSeconds = .objectives.maximumAutomaticEndpointWithdrawalSeconds'
 assert_rejected weak-rollout-objective '.objectives.minimumRolloutSuccessRatio = 0.5'
 assert_rejected weak-post-rollout-objective '.objectives.minimumPostRolloutSuccessRatio = 0.5'
 assert_rejected weak-rollback-objective '.objectives.minimumRollbackSuccessRatio = 0.5'
@@ -80,11 +87,22 @@ assert_rejected weak-transition-objective '.objectives.minimumTransitionSuccessR
 assert_rejected weak-abrupt-transition-objective '.objectives.minimumAbruptTransitionSuccessRatio = 0.5'
 assert_rejected weak-abrupt-degraded-objective '.objectives.minimumAbruptDegradedSuccessRatio = 0.5'
 assert_rejected weak-abrupt-recovered-objective '.objectives.minimumAbruptRecoveredSuccessRatio = 0.5'
+assert_rejected weak-automatic-transition-objective '.objectives.minimumAutomaticTransitionSuccessRatio = 0.5'
+assert_rejected weak-automatic-degraded-objective '.objectives.minimumAutomaticDegradedSuccessRatio = 0.5'
+assert_rejected weak-automatic-recovered-objective '.objectives.minimumAutomaticRecoveredSuccessRatio = 0.5'
 assert_rejected long-abrupt-transition-p99 '.objectives.maximumAbruptTransitionP99Millis = 10000'
 assert_rejected long-abrupt-degraded-p99 '.objectives.maximumAbruptDegradedP99Millis = 10000'
+assert_rejected long-automatic-transition-p99 '.objectives.maximumAutomaticTransitionP99Millis = 10000'
+assert_rejected long-automatic-degraded-p99 '.objectives.maximumAutomaticDegradedP99Millis = 10000'
 assert_rejected long-abrupt-endpoint-withdrawal '.objectives.maximumAbruptEndpointWithdrawalSeconds = 60'
 assert_rejected long-recovery '.objectives.maximumRecoverySeconds = 600'
 assert_rejected long-abrupt-recovery '.objectives.maximumAbruptRecoverySeconds = 600'
+assert_rejected long-automatic-node-detection '.objectives.maximumAutomaticNodeDetectionSeconds = 60'
+assert_rejected long-automatic-endpoint-withdrawal '.objectives.maximumAutomaticEndpointWithdrawalSeconds = 60'
+assert_rejected long-automatic-pod-eviction '.objectives.maximumAutomaticPodEvictionSeconds = 180'
+assert_rejected long-automatic-recovery '.objectives.maximumAutomaticRecoverySeconds = 600'
+assert_rejected unordered-automatic-node-and-endpoint-bounds '.objectives.maximumAutomaticNodeDetectionSeconds = .objectives.maximumAutomaticEndpointWithdrawalSeconds'
+assert_rejected unordered-automatic-endpoint-and-eviction-bounds '.objectives.maximumAutomaticEndpointWithdrawalSeconds = .objectives.maximumAutomaticPodEvictionSeconds'
 assert_rejected wrong-tls-hostname '.tlsRotation.hostname = "production.example.com"'
 assert_rejected wrong-baseline-tls-secret '.tlsRotation.baselineSecret = "production-server-tls"'
 assert_rejected unchanged-tls-secret '.tlsRotation.candidateSecret = .tlsRotation.baselineSecret'
@@ -94,4 +112,4 @@ assert_rejected wrong-candidate-api-key-secret '.apiKeyRotation.candidateSecret
 assert_rejected unchanged-overlap-api-key-secret '.apiKeyRotation.overlapSecret = .apiKeyRotation.baselineSecret'
 assert_rejected unchanged-candidate-api-key-secret '.apiKeyRotation.candidateSecret = .apiKeyRotation.baselineSecret'
 
-printf 'Kubernetes topology contract rejected 65 unsafe profiles without creating a cluster.\n'
+printf 'Kubernetes topology contract rejected 83 unsafe profiles without creating a cluster.\n'

scripts/bench/kubernetes-topology-profile.example.json

@@ -1,5 +1,5 @@
 {
-  "schemaVersion": 6,
+  "schemaVersion": 7,
   "profileId": "local-kind-two-zone-smoke",
   "review": {
     "status": "example"
@@ -13,7 +13,9 @@
     "zones": 2,
     "namespace": "lbp-kubernetes-smoke",
     "hostPort": 18460,
-    "nodePort": 30443
+    "nodePort": 30443,
+    "nodeMonitorGracePeriodSeconds": 20,
+    "unreachableTolerationSeconds": 10
   },
   "workload": {
     "connectionMode": "close-per-request",
@@ -34,7 +36,10 @@
     "recoveredSeconds": 8,
     "abruptTransitionSeconds": 45,
     "abruptDegradedSeconds": 8,
-    "abruptRecoveredSeconds": 8
+    "abruptRecoveredSeconds": 8,
+    "automaticTransitionSeconds": 60,
+    "automaticDegradedSeconds": 8,
+    "automaticRecoveredSeconds": 8
   },
   "objectives": {
     "minimumBaselineSuccessRatio": 0.999,
@@ -54,17 +59,26 @@
     "minimumAbruptTransitionSuccessRatio": 0.90,
     "minimumAbruptDegradedSuccessRatio": 0.95,
     "minimumAbruptRecoveredSuccessRatio": 0.999,
+    "minimumAutomaticTransitionSuccessRatio": 0.80,
+    "minimumAutomaticDegradedSuccessRatio": 0.95,
+    "minimumAutomaticRecoveredSuccessRatio": 0.999,
     "maximumP99Millis": 1500,
     "maximumAbruptTransitionP99Millis": 5500,
     "maximumAbruptDegradedP99Millis": 5500,
+    "maximumAutomaticTransitionP99Millis": 5500,
+    "maximumAutomaticDegradedP99Millis": 5500,
     "maximumRolloutSeconds": 75,
     "maximumRollbackSeconds": 75,
     "maximumCertificateRotationSeconds": 75,
     "maximumCertificateRollbackSeconds": 75,
     "maximumApiKeyTransitionSeconds": 75,
     "maximumRecoverySeconds": 180,
     "maximumAbruptEndpointWithdrawalSeconds": 15,
-    "maximumAbruptRecoverySeconds": 180
+    "maximumAbruptRecoverySeconds": 180,
+    "maximumAutomaticNodeDetectionSeconds": 30,
+    "maximumAutomaticEndpointWithdrawalSeconds": 40,
+    "maximumAutomaticPodEvictionSeconds": 90,
+    "maximumAutomaticRecoverySeconds": 180
   },
   "tlsRotation": {
     "hostname": "lbp-kubernetes.local",