diff --git a/deploy/kubernetes/kind-cluster.yaml b/deploy/kubernetes/kind-cluster.yaml index 2be3d00f..c7c73c98 100644 --- a/deploy/kubernetes/kind-cluster.yaml +++ b/deploy/kubernetes/kind-cluster.yaml @@ -6,6 +6,14 @@ nodes: - role: control-plane image: kindest/node:v1.34.3@sha256:08497ee19eace7b4b5348db5c6a1591d7752b164530a36f855cb0f2bdcbadd48 kubeadmConfigPatches: + - | + # kind 0.31.0 uses kubeadm v1beta3 for Kubernetes 1.34.x. + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + controllerManager: + extraArgs: + node-monitor-period: 2s + node-monitor-grace-period: 20s - | apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index dc354c6d..50357b5f 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -61,6 +61,15 @@ spec: enableServiceLinks: false nodeSelector: loadbalancerpro.io/qualification-worker: "true" + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 10 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 10 topologySpreadConstraints: - maxSkew: 1 minDomains: 2 @@ -140,6 +149,15 @@ spec: enableServiceLinks: false nodeSelector: loadbalancerpro.io/qualification-worker: "true" + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 10 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 10 topologySpreadConstraints: - maxSkew: 1 minDomains: 2 @@ -230,6 +248,15 @@ spec: terminationGracePeriodSeconds: 45 nodeSelector: loadbalancerpro.io/qualification-worker: "true" + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 10 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 10 topologySpreadConstraints: - maxSkew: 1 minDomains: 2 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 8582c632..b0a1ac4b 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -144,10 +144,16 @@ after stale pooled connections have been discarded. The lane then proves two-zon Service distribution, planned worker removal, and operator-remediated no-drain worker loss and recovery. The abrupt-loss exercise forcibly stops the kind worker, confirms its container is down, applies the out-of-service `NoExecute` taint, and force-removes the three exact stateless -qualification pods from the API. The disposable cluster also pins immediate EndpointSlice-triggered iptables updates -and a one-second kube-proxy cleanup sync; operators must review the equivalent setting or managed-ingress -behavior for their environment. The candidate preserves the baseline application layers, so this does not prove -application-layer release compatibility or automatic deployment failure detection. The reviewed staging runner +qualification pods from the API. A separate no-drain kill then permits no post-failure Kubernetes mutation: the pinned +controller-manager must detect the lost node, EndpointSlices must withdraw its endpoints, and the bounded `NoExecute` +tolerations must initiate eviction for its three pods. The evidence accepts API removal or a deletion timestamp because +an unavailable kubelet can leave the old objects terminating until it returns; fresh identities must still recover across +both zones. The disposable cluster pins a +twenty-second node-monitor grace period, ten-second unreachable tolerations, immediate EndpointSlice-triggered iptables +updates, and a one-second kube-proxy cleanup sync; operators must review equivalent managed-control-plane and ingress +behavior instead of copying those lab timings. Its controller-detected transition has an 80% success floor that includes +the grace interval, followed by a 95% degraded floor and 99.9% recovered floor. The candidate preserves the baseline application layers, so this does not +prove application-layer release compatibility or deployment-equivalent infrastructure failure timing. The reviewed staging runner separately validates the external target's digest, replicas, zones, resources, configuration, ingress, metrics, drain, and transitions; local proof does not establish registry integrity, deployment capacity, external ingress behavior, or production readiness. diff --git a/docs/LOAD_BALANCER_BUILD_OUT.md b/docs/LOAD_BALANCER_BUILD_OUT.md index 650340c0..f4e33123 100644 --- a/docs/LOAD_BALANCER_BUILD_OUT.md +++ b/docs/LOAD_BALANCER_BUILD_OUT.md @@ -175,17 +175,23 @@ and second replica must recover inside the bound, and both recovered replicas an then forcibly stops that recovered worker without a drain, confirms the container is down, and applies the documented out-of-service `NoExecute` remediation. It force-removes the three exact stateless workload pods from the API, bounds endpoint withdrawal, proves degraded traffic, rejects the failed pod identity after recovery, and requires both -recovered replicas and backends to serve new traffic. The disposable cluster pins iptables-mode kube-proxy -to immediate EndpointSlice-triggered updates and a one-second cleanup sync; deployment environments must review the -equivalent Service/ingress failure-detection and reconciliation behavior. Its abrupt transition and degraded phases -bound stale conntrack impact at 90% and 95% success with 5.5-second p99 ceilings; recovered traffic must return to the -normal 99.9% success and 1.5-second p99 objectives. The ten-second endpoint drain exceeds the five-second qualification +recovered replicas and backends to serve new traffic. A second forced stop permits no post-failure Kubernetes mutation: +the test-configured controller-manager must detect the lost node, EndpointSlices must withdraw its endpoints, the bounded +`NoExecute` tolerations must initiate eviction for its three pods, and fresh pod identities must recover across both zones. +Eviction evidence accepts API removal or a deletion timestamp because an unavailable kubelet can leave the old objects +terminating until the node returns. The disposable +cluster pins a twenty-second node-monitor grace period, ten-second unreachable tolerations, iptables-mode kube-proxy with +immediate EndpointSlice-triggered updates, and a one-second cleanup sync; deployment environments must review equivalent +managed-control-plane and Service/ingress reconciliation behavior. Its abrupt transition and degraded phases +bound stale conntrack impact at 90% and 95% success with 5.5-second p99 ceilings. The controller-detected transition has +an 80% floor that includes the configured node-monitor grace interval, followed by a 95% degraded floor; recovered +traffic must return to the normal 99.9% success and 1.5-second p99 objectives. The ten-second endpoint drain exceeds the five-second qualification client timeout, and the 45-second termination grace contains the 30-second application shutdown bound. The candidate has a distinct local image content ID but preserves the baseline application layers, so it proves Kubernetes transition and rollback mechanics rather than compatibility between application releases. The reviewed deployment ingress, deployment-equivalent resources, registry -digest transition, and automatic infrastructure-failure detection remain staging gates. +digest transition, and deployment-equivalent infrastructure-failure timing remain staging gates. ### 4. Stage The Rollout And Rollback diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 861b1754..c5f05aff 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -141,10 +141,17 @@ tests the one-replica degraded service, requires both recovered replicas and bac forcibly stops that recovered worker without a drain. After confirming the worker container is down, it applies Kubernetes' out-of-service `NoExecute` remediation and force-removes the three exact stateless workload pods from the API, bounds endpoint withdrawal, proves degraded traffic, and requires fresh pod identity, two-zone placement, and traffic -distribution after recovery. The lab cluster pins iptables kube-proxy to immediate EndpointSlice-triggered updates and a -one-second cleanup sync so the Service failover objective is executable and recorded. The abrupt transition and degraded -windows retain bounded 90% and 95% success floors with 5.5-second p99 ceilings for stale conntrack paths; recovered -traffic returns to the normal 99.9% success and 1.5-second p99 objectives. Rollouts drain endpoints for ten seconds, +distribution after recovery. It then repeats a no-drain worker kill without any post-failure Kubernetes mutation. The +controller-manager must detect the lost node, EndpointSlices must withdraw its endpoints, bounded `NoExecute` tolerations +must initiate eviction for the three dead-node pods, and recovery must use fresh pod identities across both zones. The +evidence accepts either API removal or a deletion timestamp because an unavailable kubelet can leave evicted pod objects +terminating until the node returns. The lab cluster pins a +twenty-second node-monitor grace period, ten-second unreachable tolerations, iptables kube-proxy with immediate +EndpointSlice-triggered updates, and a one-second cleanup sync so the automatic Service failover objective is executable +and recorded. Both abrupt transition and degraded windows retain bounded 90% and 95% success floors with 5.5-second p99 +ceilings for stale conntrack paths. The controller-detected transition has a separate 80% floor to include the configured +node-monitor grace interval, while its degraded phase retains 95%; recovered traffic returns to the normal 99.9% success +and 1.5-second p99 objectives. Rollouts drain endpoints for ten seconds, longer than the five-second qualification client timeout, while the 45-second grace period contains the application's 30-second graceful-shutdown bound: @@ -162,8 +169,8 @@ prove dynamic Secret reload or an external secret manager, and it does not deplo external certificate authority, or client trust-distribution system. Because the local candidate changes immutable proof metadata but not application layers, it does not prove application-layer release compatibility, registry integrity, -deployment capacity, external ingress behavior, automatic infrastructure-failure detection, or an authorized staging -environment. +deployment capacity, external ingress behavior, deployment-equivalent infrastructure-failure timing, or an authorized +staging environment. ## Local capacity staircase diff --git a/scripts/bench/kubernetes-topology-contract-test.sh b/scripts/bench/kubernetes-topology-contract-test.sh index 42fc0438..13bb5c95 100644 --- a/scripts/bench/kubernetes-topology-contract-test.sh +++ b/scripts/bench/kubernetes-topology-contract-test.sh @@ -14,7 +14,7 @@ trap cleanup EXIT bash -n "$runner" bash "$runner" --mode validate --profile "$profile" > "$work_dir/valid.log" -grep -Fq 'service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery' \ +grep -Fq 'service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery automatic-node-detection automatic-endpoint-withdrawal automatic-pod-eviction automatic-recovery' \ "$work_dir/valid.log" assert_rejected() { @@ -33,12 +33,15 @@ assert_rejected legacy-schema '.schemaVersion = 1' assert_rejected older-schema '.schemaVersion = 3' assert_rejected older-schema-four '.schemaVersion = 4' assert_rejected previous-schema '.schemaVersion = 5' +assert_rejected previous-schema-six '.schemaVersion = 6' assert_rejected skewed-kubectl '.cluster.kubectlVersion = "v1.32.2"' assert_rejected mutable-node-image '.cluster.nodeImageDigest = "sha256:" + ("f" * 64)' assert_rejected one-worker '.cluster.workers = 1' assert_rejected one-zone '.cluster.zones = 1' assert_rejected external-namespace '.cluster.namespace = "production"' assert_rejected public-port '.cluster.hostPort = 443' +assert_rejected slow-node-monitor '.cluster.nodeMonitorGracePeriodSeconds = 50' +assert_rejected slow-unreachable-toleration '.cluster.unreachableTolerationSeconds = 300' assert_rejected sticky-connection-mode '.workload.connectionMode = "keep-alive"' assert_rejected low-rate '.workload.ratePerSecond = 1' assert_rejected short-rollout '.workload.rolloutSeconds = 10' @@ -61,6 +64,10 @@ assert_rejected short-abrupt-transition '.workload.abruptTransitionSeconds = 5' assert_rejected short-abrupt-degraded '.workload.abruptDegradedSeconds = 1' assert_rejected short-abrupt-recovered '.workload.abruptRecoveredSeconds = 1' assert_rejected abrupt-window-too-short '.workload.abruptTransitionSeconds = .objectives.maximumAbruptEndpointWithdrawalSeconds' +assert_rejected short-automatic-transition '.workload.automaticTransitionSeconds = 10' +assert_rejected short-automatic-degraded '.workload.automaticDegradedSeconds = 1' +assert_rejected short-automatic-recovered '.workload.automaticRecoveredSeconds = 1' +assert_rejected automatic-window-too-short '.workload.automaticTransitionSeconds = .objectives.maximumAutomaticEndpointWithdrawalSeconds' assert_rejected weak-rollout-objective '.objectives.minimumRolloutSuccessRatio = 0.5' assert_rejected weak-post-rollout-objective '.objectives.minimumPostRolloutSuccessRatio = 0.5' assert_rejected weak-rollback-objective '.objectives.minimumRollbackSuccessRatio = 0.5' @@ -80,11 +87,22 @@ assert_rejected weak-transition-objective '.objectives.minimumTransitionSuccessR assert_rejected weak-abrupt-transition-objective '.objectives.minimumAbruptTransitionSuccessRatio = 0.5' assert_rejected weak-abrupt-degraded-objective '.objectives.minimumAbruptDegradedSuccessRatio = 0.5' assert_rejected weak-abrupt-recovered-objective '.objectives.minimumAbruptRecoveredSuccessRatio = 0.5' +assert_rejected weak-automatic-transition-objective '.objectives.minimumAutomaticTransitionSuccessRatio = 0.5' +assert_rejected weak-automatic-degraded-objective '.objectives.minimumAutomaticDegradedSuccessRatio = 0.5' +assert_rejected weak-automatic-recovered-objective '.objectives.minimumAutomaticRecoveredSuccessRatio = 0.5' assert_rejected long-abrupt-transition-p99 '.objectives.maximumAbruptTransitionP99Millis = 10000' assert_rejected long-abrupt-degraded-p99 '.objectives.maximumAbruptDegradedP99Millis = 10000' +assert_rejected long-automatic-transition-p99 '.objectives.maximumAutomaticTransitionP99Millis = 10000' +assert_rejected long-automatic-degraded-p99 '.objectives.maximumAutomaticDegradedP99Millis = 10000' assert_rejected long-abrupt-endpoint-withdrawal '.objectives.maximumAbruptEndpointWithdrawalSeconds = 60' assert_rejected long-recovery '.objectives.maximumRecoverySeconds = 600' assert_rejected long-abrupt-recovery '.objectives.maximumAbruptRecoverySeconds = 600' +assert_rejected long-automatic-node-detection '.objectives.maximumAutomaticNodeDetectionSeconds = 60' +assert_rejected long-automatic-endpoint-withdrawal '.objectives.maximumAutomaticEndpointWithdrawalSeconds = 60' +assert_rejected long-automatic-pod-eviction '.objectives.maximumAutomaticPodEvictionSeconds = 180' +assert_rejected long-automatic-recovery '.objectives.maximumAutomaticRecoverySeconds = 600' +assert_rejected unordered-automatic-node-and-endpoint-bounds '.objectives.maximumAutomaticNodeDetectionSeconds = .objectives.maximumAutomaticEndpointWithdrawalSeconds' +assert_rejected unordered-automatic-endpoint-and-eviction-bounds '.objectives.maximumAutomaticEndpointWithdrawalSeconds = .objectives.maximumAutomaticPodEvictionSeconds' assert_rejected wrong-tls-hostname '.tlsRotation.hostname = "production.example.com"' assert_rejected wrong-baseline-tls-secret '.tlsRotation.baselineSecret = "production-server-tls"' assert_rejected unchanged-tls-secret '.tlsRotation.candidateSecret = .tlsRotation.baselineSecret' @@ -94,4 +112,4 @@ assert_rejected wrong-candidate-api-key-secret '.apiKeyRotation.candidateSecret assert_rejected unchanged-overlap-api-key-secret '.apiKeyRotation.overlapSecret = .apiKeyRotation.baselineSecret' assert_rejected unchanged-candidate-api-key-secret '.apiKeyRotation.candidateSecret = .apiKeyRotation.baselineSecret' -printf 'Kubernetes topology contract rejected 65 unsafe profiles without creating a cluster.\n' +printf 'Kubernetes topology contract rejected 83 unsafe profiles without creating a cluster.\n' diff --git a/scripts/bench/kubernetes-topology-profile.example.json b/scripts/bench/kubernetes-topology-profile.example.json index f9f78b59..570f5608 100644 --- a/scripts/bench/kubernetes-topology-profile.example.json +++ b/scripts/bench/kubernetes-topology-profile.example.json @@ -1,5 +1,5 @@ { - "schemaVersion": 6, + "schemaVersion": 7, "profileId": "local-kind-two-zone-smoke", "review": { "status": "example" @@ -13,7 +13,9 @@ "zones": 2, "namespace": "lbp-kubernetes-smoke", "hostPort": 18460, - "nodePort": 30443 + "nodePort": 30443, + "nodeMonitorGracePeriodSeconds": 20, + "unreachableTolerationSeconds": 10 }, "workload": { "connectionMode": "close-per-request", @@ -34,7 +36,10 @@ "recoveredSeconds": 8, "abruptTransitionSeconds": 45, "abruptDegradedSeconds": 8, - "abruptRecoveredSeconds": 8 + "abruptRecoveredSeconds": 8, + "automaticTransitionSeconds": 60, + "automaticDegradedSeconds": 8, + "automaticRecoveredSeconds": 8 }, "objectives": { "minimumBaselineSuccessRatio": 0.999, @@ -54,9 +59,14 @@ "minimumAbruptTransitionSuccessRatio": 0.90, "minimumAbruptDegradedSuccessRatio": 0.95, "minimumAbruptRecoveredSuccessRatio": 0.999, + "minimumAutomaticTransitionSuccessRatio": 0.80, + "minimumAutomaticDegradedSuccessRatio": 0.95, + "minimumAutomaticRecoveredSuccessRatio": 0.999, "maximumP99Millis": 1500, "maximumAbruptTransitionP99Millis": 5500, "maximumAbruptDegradedP99Millis": 5500, + "maximumAutomaticTransitionP99Millis": 5500, + "maximumAutomaticDegradedP99Millis": 5500, "maximumRolloutSeconds": 75, "maximumRollbackSeconds": 75, "maximumCertificateRotationSeconds": 75, @@ -64,7 +74,11 @@ "maximumApiKeyTransitionSeconds": 75, "maximumRecoverySeconds": 180, "maximumAbruptEndpointWithdrawalSeconds": 15, - "maximumAbruptRecoverySeconds": 180 + "maximumAbruptRecoverySeconds": 180, + "maximumAutomaticNodeDetectionSeconds": 30, + "maximumAutomaticEndpointWithdrawalSeconds": 40, + "maximumAutomaticPodEvictionSeconds": 90, + "maximumAutomaticRecoverySeconds": 180 }, "tlsRotation": { "hostname": "lbp-kubernetes.local", diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index e19c2016..d4ba2f51 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -25,7 +25,7 @@ for required_file in "$cluster_config" "$workload_manifest" "$candidate_dockerfi done jq -e ' - .schemaVersion == 6 + .schemaVersion == 7 and (.profileId | type == "string" and test("^[a-z0-9][a-z0-9._-]{0,62}$")) and .review.status == "example" and .cluster.kindVersion == "v0.31.0" @@ -37,6 +37,8 @@ jq -e ' and .cluster.namespace == "lbp-kubernetes-smoke" and .cluster.hostPort == 18460 and .cluster.nodePort == 30443 + and .cluster.nodeMonitorGracePeriodSeconds == 20 + and .cluster.unreachableTolerationSeconds == 10 and .workload.connectionMode == "close-per-request" and (.workload.ratePerSecond | type == "number" and . >= 10 and . <= 500 and floor == .) and (.workload.baselineSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) @@ -56,6 +58,9 @@ jq -e ' and (.workload.abruptTransitionSeconds | type == "number" and . >= 15 and . <= 120 and floor == .) and (.workload.abruptDegradedSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.abruptRecoveredSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) + and (.workload.automaticTransitionSeconds | type == "number" and . >= 30 and . <= 120 and floor == .) + and (.workload.automaticDegradedSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) + and (.workload.automaticRecoveredSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.objectives.minimumBaselineSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumRolloutSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumPostRolloutSuccessRatio | type == "number" and . >= 0.95 and . <= 1) @@ -73,9 +78,14 @@ jq -e ' and (.objectives.minimumAbruptTransitionSuccessRatio | type == "number" and . >= 0.90 and . <= 1) and (.objectives.minimumAbruptDegradedSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumAbruptRecoveredSuccessRatio | type == "number" and . >= 0.95 and . <= 1) + and (.objectives.minimumAutomaticTransitionSuccessRatio | type == "number" and . >= 0.80 and . <= 1) + and (.objectives.minimumAutomaticDegradedSuccessRatio | type == "number" and . >= 0.95 and . <= 1) + and (.objectives.minimumAutomaticRecoveredSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.maximumP99Millis | type == "number" and . >= 100 and . <= 5000 and floor == .) and (.objectives.maximumAbruptTransitionP99Millis | type == "number" and . >= 1000 and . <= 6000 and floor == .) and (.objectives.maximumAbruptDegradedP99Millis | type == "number" and . >= 1000 and . <= 6000 and floor == .) + and (.objectives.maximumAutomaticTransitionP99Millis | type == "number" and . >= 1000 and . <= 6000 and floor == .) + and (.objectives.maximumAutomaticDegradedP99Millis | type == "number" and . >= 1000 and . <= 6000 and floor == .) and (.objectives.maximumRolloutSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.objectives.maximumRollbackSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.objectives.maximumCertificateRotationSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) @@ -84,12 +94,19 @@ jq -e ' and (.objectives.maximumRecoverySeconds | type == "number" and . >= 30 and . <= 300 and floor == .) and (.objectives.maximumAbruptEndpointWithdrawalSeconds | type == "number" and . >= 5 and . <= 30 and floor == .) and (.objectives.maximumAbruptRecoverySeconds | type == "number" and . >= 30 and . <= 300 and floor == .) + and (.objectives.maximumAutomaticNodeDetectionSeconds | type == "number" and . >= 10 and . <= 30 and floor == .) + and (.objectives.maximumAutomaticEndpointWithdrawalSeconds | type == "number" and . >= 10 and . <= 45 and floor == .) + and (.objectives.maximumAutomaticPodEvictionSeconds | type == "number" and . >= 30 and . <= 120 and floor == .) + and (.objectives.maximumAutomaticRecoverySeconds | type == "number" and . >= 30 and . <= 300 and floor == .) and .workload.rolloutSeconds >= (.objectives.maximumRolloutSeconds + 5) and .workload.rollbackSeconds >= (.objectives.maximumRollbackSeconds + 5) and .workload.certificateRotationSeconds >= (.objectives.maximumCertificateRotationSeconds + 5) and .workload.certificateRollbackSeconds >= (.objectives.maximumCertificateRollbackSeconds + 5) and .workload.apiKeyTransitionSeconds >= (.objectives.maximumApiKeyTransitionSeconds + 5) and .workload.abruptTransitionSeconds >= (.objectives.maximumAbruptEndpointWithdrawalSeconds + 5) + and .workload.automaticTransitionSeconds >= (.objectives.maximumAutomaticEndpointWithdrawalSeconds + 5) + and .objectives.maximumAutomaticNodeDetectionSeconds < .objectives.maximumAutomaticEndpointWithdrawalSeconds + and .objectives.maximumAutomaticEndpointWithdrawalSeconds < .objectives.maximumAutomaticPodEvictionSeconds and .tlsRotation.hostname == "lbp-kubernetes.local" and .tlsRotation.baselineSecret == "loadbalancerpro-server-tls-a" and .tlsRotation.candidateSecret == "loadbalancerpro-server-tls-b" @@ -107,6 +124,10 @@ for invariant in \ 'containerPort: 30443' \ 'hostPort: 18460' \ 'kind: KubeProxyConfiguration' \ + 'apiVersion: kubeadm.k8s.io/v1beta3' \ + 'kind: ClusterConfiguration' \ + 'node-monitor-period: 2s' \ + 'node-monitor-grace-period: 20s' \ 'mode: iptables' \ 'minSyncPeriod: 0s' \ 'syncPeriod: 1s'; do @@ -134,6 +155,9 @@ for invariant in \ 'LBP_RETRY_MAX_ATTEMPTS: "3"' \ 'LBP_RETRY_BUDGET_PERCENT: "100"' \ 'LBP_RETRY_NON_IDEMPOTENT: "false"' \ + 'key: node.kubernetes.io/not-ready' \ + 'key: node.kubernetes.io/unreachable' \ + 'tolerationSeconds: 10' \ 'path: loadbalancerpro.api.rotation-key'; do grep -Fq "$invariant" "$workload_manifest" || { echo "Kubernetes workload is missing: $invariant" >&2; exit 2; } done @@ -144,7 +168,7 @@ fi if [[ "$mode" == "validate" ]]; then printf 'Validated disposable two-worker/two-zone Kubernetes topology contract %s.\n' "$(jq -r '.profileId' "$profile")" - printf 'Validated proof cases: service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery\n' + printf 'Validated proof cases: service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery automatic-node-detection automatic-endpoint-withdrawal automatic-pod-eviction automatic-recovery\n' exit 0 fi @@ -289,6 +313,14 @@ else fi [[ "$(kubectl config current-context)" == "kind-$cluster_name" ]] || { echo "Unexpected Kubernetes context" >&2; exit 1; } kubectl wait --for=condition=Ready nodes --all --timeout=120s +controller_manager_command="$(kubectl get pod --namespace kube-system \ + -l component=kube-controller-manager -o json | jq -r '.items[0].spec.containers[0].command[]')" +for invariant in '--node-monitor-period=2s' '--node-monitor-grace-period=20s'; do + grep -Fxq -- "$invariant" <<< "$controller_manager_command" || { + echo "Live kube-controller-manager command is missing: $invariant" >&2; exit 1; + } +done +printf '%s\n' "$controller_manager_command" > "$output_dir/kube-controller-manager-command.txt" kube_proxy_config="$(kubectl get configmap kube-proxy --namespace kube-system -o json \ | jq -r '.data["config.conf"]')" for invariant in 'mode: iptables' 'minSyncPeriod: 0s' 'syncPeriod: 1s'; do @@ -360,6 +392,17 @@ chmod 0600 "$api_key_file" "$candidate_api_key_file" "$empty_rotation_key_file" "$tls_dir"/* "$candidate_tls_dir"/* "$tls_trust_bundle" kubectl apply --server-side --field-manager=loadbalancerpro-qualification -f "$workload_manifest" +kubectl get deployment backend-a backend-b loadbalancerpro --namespace "$namespace" -o json \ + | jq '{items: [.items[] | {deployment: .metadata.name, + tolerations: .spec.template.spec.tolerations}]}' > "$output_dir/node-failure-policy.json" +jq -e '(.items | length) == 3 and all(.items[]; + ([.tolerations[] | select(.key == "node.kubernetes.io/not-ready" + and .operator == "Exists" and .effect == "NoExecute" and .tolerationSeconds == 10)] | length) == 1 + and ([.tolerations[] | select(.key == "node.kubernetes.io/unreachable" + and .operator == "Exists" and .effect == "NoExecute" and .tolerationSeconds == 10)] | length) == 1)' \ + "$output_dir/node-failure-policy.json" >/dev/null || { + echo "Qualification Deployments do not share the bounded unreachable-node eviction policy" >&2; exit 1; +} create_immutable_api_key_secret() { local secret_name="$1" primary_file="$2" rotation_file="$3" kubectl create secret generic "$secret_name" --namespace "$namespace" \ @@ -445,6 +488,14 @@ abrupt_source_pod_count() { '[.items[] | select(.metadata.name as $name | $names | index($name))] | length' } +automatic_active_source_pod_count() { + kubectl get pod --namespace "$namespace" -o json \ + | jq --argjson names "$automatic_source_pod_names_json" \ + '[.items[] + | select(.metadata.name as $name | $names | index($name)) + | select(.metadata.deletionTimestamp == null)] | length' +} + wait_for_count() { local description="$1" expected="$2" command_name="$3" timeout_seconds="$4" local deadline=$((SECONDS + timeout_seconds)) actual=unknown @@ -457,6 +508,19 @@ wait_for_count() { return 1 } +wait_for_node_unavailable() { + local node="$1" timeout_seconds="$2" + local deadline=$((SECONDS + timeout_seconds)) actual=unknown + while (( SECONDS < deadline )); do + actual="$(kubectl get node "$node" -o json \ + | jq -r '[.status.conditions[] | select(.type == "Ready")][0].status // "Missing"')" + case "$actual" in False|Unknown) automatic_node_ready_status="$actual"; return 0 ;; esac + sleep 2 + done + echo "Timed out waiting for Kubernetes to detect $node as unavailable; observed Ready=$actual" >&2 + return 1 +} + wait_for_backend_endpoint_count() { local service_name="$1" expected="$2" timeout_seconds="$3" local deadline=$((SECONDS + timeout_seconds)) actual=unknown @@ -1654,6 +1718,165 @@ jq -e '(.pods | length) == 2 echo "Both recovered proxies and both backends must serve traffic after abrupt loss" >&2; exit 1; } +assert_no_container_restarts pre-automatic-worker-loss +automatic_ready_proxy_pods_json="$(kubectl get pod --namespace "$namespace" \ + -l app.kubernetes.io/name=loadbalancerpro -o json | jq '[.items[] + | select(.metadata.deletionTimestamp == null) + | select(.status.phase == "Running") + | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))]')" +[[ "$(jq 'length' <<< "$automatic_ready_proxy_pods_json")" == 2 ]] || { + echo "Expected two ready proxies before automatic worker-loss detection" >&2; exit 1; +} +automatic_node="$(jq -r 'sort_by(.spec.nodeName)[-1].spec.nodeName' \ + <<< "$automatic_ready_proxy_pods_json")" +[[ "$automatic_node" == "${cluster_name}-worker" || "$automatic_node" == "${cluster_name}-worker2" ]] || { + echo "Refusing to stop unexpected automatic-loss node $automatic_node" >&2; exit 1; +} +automatic_failed_proxy_uid="$(jq -r --arg node "$automatic_node" \ + '.[] | select(.spec.nodeName == $node) | .metadata.uid' <<< "$automatic_ready_proxy_pods_json")" +[[ "$automatic_failed_proxy_uid" =~ ^[0-9a-f-]{36}$ ]] || { + echo "Unable to bind the proxy pod identity for automatic worker-loss detection" >&2; exit 1; +} +automatic_node_pods_json="$(kubectl get pod --namespace "$namespace" -o json \ + | jq --arg node "$automatic_node" '[.items[] + | select(.metadata.deletionTimestamp == null) + | select(.spec.nodeName == $node) + | select(.metadata.labels["app.kubernetes.io/name"] == "loadbalancerpro" + or .metadata.labels["app.kubernetes.io/name"] == "fixture-backend")]')" +[[ "$(jq 'length' <<< "$automatic_node_pods_json")" == 3 ]] || { + echo "Expected exactly one proxy and two backend pods on the automatic-loss worker" >&2; exit 1; +} +automatic_source_pod_names_json="$(jq '[.[].metadata.name] | sort' <<< "$automatic_node_pods_json")" + +automatic_transition_seconds="$(jq -r '.workload.automaticTransitionSeconds' "$profile")" +maximum_automatic_node_detection_seconds="$(jq -r \ + '.objectives.maximumAutomaticNodeDetectionSeconds' "$profile")" +maximum_automatic_endpoint_withdrawal_seconds="$(jq -r \ + '.objectives.maximumAutomaticEndpointWithdrawalSeconds' "$profile")" +maximum_automatic_pod_eviction_seconds="$(jq -r \ + '.objectives.maximumAutomaticPodEvictionSeconds' "$profile")" +vegeta attack -duration="${automatic_transition_seconds}s" -rate="${rate}/s" -timeout=5s \ + -keepalive=false -http2=false -root-certs="$tls_trust_bundle" -targets="$targets" \ + > "$work_dir/automatic-transition.bin" & +attack_pid=$! +sleep 3 +automatic_failure_started_epoch="$(date +%s)" +docker kill "$automatic_node" >/dev/null +stopped_node="$automatic_node" +[[ "$(docker inspect --format '{{.State.Running}}' "$automatic_node")" == false ]] || { + echo "Automatic-loss worker container is still running" >&2; exit 1; +} +wait_for_node_unavailable "$automatic_node" "$maximum_automatic_node_detection_seconds" +automatic_node_detection_seconds=$(( $(date +%s) - automatic_failure_started_epoch )) +(( automatic_node_detection_seconds <= maximum_automatic_node_detection_seconds )) || { + echo "Automatic worker-loss node detection exceeded the objective" >&2; exit 1; +} +wait_for_count 'ready Service endpoints after automatic worker loss' 1 ready_endpoint_count \ + "$maximum_automatic_endpoint_withdrawal_seconds" +automatic_endpoint_withdrawal_seconds=$(( $(date +%s) - automatic_failure_started_epoch )) +(( automatic_endpoint_withdrawal_seconds <= maximum_automatic_endpoint_withdrawal_seconds )) || { + echo "Automatic worker-loss endpoint withdrawal exceeded the objective" >&2; exit 1; +} +wait_for_count 'automatic-loss source pods still active in the API' 0 automatic_active_source_pod_count \ + "$maximum_automatic_pod_eviction_seconds" +automatic_pod_eviction_seconds=$(( $(date +%s) - automatic_failure_started_epoch )) +(( automatic_pod_eviction_seconds <= maximum_automatic_pod_eviction_seconds )) || { + echo "Automatic worker-loss pod eviction exceeded the objective" >&2; exit 1; +} +automatic_source_pod_evictions_json="$(kubectl get pod --namespace "$namespace" -o json \ + | jq --argjson names "$automatic_source_pod_names_json" ' + .items as $items + | [$names[] as $name + | ($items | map(select(.metadata.name == $name)) | .[0] // null) as $pod + | {name: $name, + observedInApi: ($pod != null), + deletionTimestamp: ($pod.metadata.deletionTimestamp // null), + evictionObserved: (($pod == null) or ($pod.metadata.deletionTimestamp != null))}]')" +jq -e 'length == 3 and all(.[]; .evictionObserved)' \ + <<< "$automatic_source_pod_evictions_json" >/dev/null || { + echo "Automatic worker-loss source pod eviction evidence was incomplete" >&2; exit 1; +} +printf '%s\n' "$automatic_source_pod_evictions_json" \ + > "$output_dir/automatic-source-pod-evictions.json" +jq -n \ + --argjson nodeDetectionSeconds "$automatic_node_detection_seconds" \ + --argjson endpointWithdrawalSeconds "$automatic_endpoint_withdrawal_seconds" \ + --argjson podEvictionSeconds "$automatic_pod_eviction_seconds" \ + '{nodeDetectionSeconds: $nodeDetectionSeconds, + endpointWithdrawalSeconds: $endpointWithdrawalSeconds, + podEvictionSeconds: $podEvictionSeconds}' \ + > "$output_dir/automatic-failure-timings.json" +wait_for_count 'ready proxy replicas after automatic worker loss' 1 ready_proxy_count 10 +if ! wait "$attack_pid"; then + attack_pid="" + echo "Automatic worker-loss transition traffic attack failed" >&2 + exit 1 +fi +attack_pid="" +report_attack automatic-transition \ + "$(jq -r '.objectives.minimumAutomaticTransitionSuccessRatio' "$profile")" \ + "$(jq -r '.objectives.maximumAutomaticTransitionP99Millis' "$profile")" +capture_state automatic-degraded +assert_no_container_restarts automatic-degraded +run_attack automatic-degraded "$(jq -r '.workload.automaticDegradedSeconds' "$profile")" \ + "$(jq -r '.objectives.minimumAutomaticDegradedSuccessRatio' "$profile")" "$targets" \ + "$(jq -r '.objectives.maximumAutomaticDegradedP99Millis' "$profile")" + +automatic_recovery_started_epoch="$(date +%s)" +docker start "$stopped_node" >/dev/null +stopped_node="" +maximum_automatic_recovery_seconds="$(jq -r '.objectives.maximumAutomaticRecoverySeconds' "$profile")" +kubectl wait --for=condition=Ready node/"$automatic_node" \ + --timeout="${maximum_automatic_recovery_seconds}s" +for deployment in backend-a backend-b loadbalancerpro; do + kubectl rollout status deployment/"$deployment" --namespace "$namespace" \ + --timeout="${maximum_automatic_recovery_seconds}s" +done +wait_for_count 'automatic-loss recovered proxy replicas' 2 ready_proxy_count \ + "$maximum_automatic_recovery_seconds" +wait_for_count 'automatic-loss recovered Service endpoints' 2 ready_endpoint_count \ + "$maximum_automatic_recovery_seconds" +automatic_recovery_seconds=$(( $(date +%s) - automatic_recovery_started_epoch )) +(( automatic_recovery_seconds <= maximum_automatic_recovery_seconds )) || { + echo "Automatic worker-loss recovery exceeded the objective" >&2; exit 1; +} +automatic_recovered_proxy_pods_json="$(kubectl get pod --namespace "$namespace" \ + -l app.kubernetes.io/name=loadbalancerpro -o json | jq '[.items[] + | select(.metadata.deletionTimestamp == null) + | select(.status.phase == "Running") + | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))]')" +[[ "$(jq '[.[].spec.nodeName] | unique | length' <<< "$automatic_recovered_proxy_pods_json")" == 2 ]] || { + echo "Automatic worker-loss recovery did not restore distinct workers" >&2; exit 1; +} +automatic_failed_uid_overlap="$(jq -n --arg prior "$automatic_failed_proxy_uid" \ + --argjson recovered "$automatic_recovered_proxy_pods_json" \ + '[ $recovered[].metadata.uid | select(. == $prior) ] | length')" +[[ "$automatic_failed_uid_overlap" == 0 ]] || { + echo "Automatic worker-loss recovery retained the failed worker pod UID" >&2; exit 1; +} +capture_state automatic-recovered +assert_no_container_restarts automatic-recovered +collect_distribution automatic-recovered-before false +run_attack automatic-recovered "$(jq -r '.workload.automaticRecoveredSeconds' "$profile")" \ + "$(jq -r '.objectives.minimumAutomaticRecoveredSuccessRatio' "$profile")" +collect_distribution automatic-recovered +jq -n --slurpfile before "$output_dir/automatic-recovered-before-distribution.json" \ + --slurpfile after "$output_dir/automatic-recovered-distribution.json" ' + ($before[0]) as $before | ($after[0]) as $after | + {phase: "automatic-recovered", bothRecoveredProxyReplicasServed: true, + backendARequestDelta: ($after.backendARequests - $before.backendARequests), + backendBRequestDelta: ($after.backendBRequests - $before.backendBRequests), + pods: [$after.pods[] as $current + | ($before.pods[] | select(.pod == $current.pod)) as $prior + | {pod: $current.pod, requestDelta: ($current.requests - $prior.requests)}]} + ' > "$output_dir/automatic-recovered-distribution-delta.json" +jq -e '(.pods | length) == 2 + and all(.pods[]; .requestDelta > 0) + and .backendARequestDelta > 0 + and .backendBRequestDelta > 0' "$output_dir/automatic-recovered-distribution-delta.json" >/dev/null || { + echo "Both recovered proxies and both backends must serve traffic after automatic loss" >&2; exit 1; +} + kubectl version -o json > "$output_dir/kubernetes-version.json" kind version > "$output_dir/kind-version.txt" docker image inspect "$proxy_image_id" > "$output_dir/proxy-baseline-image.json" @@ -1668,6 +1891,7 @@ post_certificate_rollback_distribution_delta_json="$(<"$output_dir/post-certific api_key_rotation_json="$(<"$output_dir/api-key-rotation.json")" recovered_distribution_delta_json="$(<"$output_dir/recovered-distribution-delta.json")" abrupt_recovered_distribution_delta_json="$(<"$output_dir/abrupt-recovered-distribution-delta.json")" +automatic_recovered_distribution_delta_json="$(<"$output_dir/automatic-recovered-distribution-delta.json")" jq -n \ --arg profileId "$profile_id" \ --arg sourceRevision "$source_revision" \ @@ -1687,6 +1911,11 @@ jq -n \ --arg abruptWorker "$abrupt_node" \ --arg abruptFailedProxyUid "$abrupt_failed_proxy_uid" \ --argjson abruptForcedPodNames "$abrupt_forced_pod_names_json" \ + --arg automaticWorker "$automatic_node" \ + --arg automaticNodeReadyStatus "$automatic_node_ready_status" \ + --arg automaticFailedProxyUid "$automatic_failed_proxy_uid" \ + --argjson automaticSourcePodNames "$automatic_source_pod_names_json" \ + --argjson automaticSourcePodEvictions "$automatic_source_pod_evictions_json" \ --argjson priorPodUids "$initial_proxy_uids_json" \ --argjson candidatePodUids "$replacement_proxy_uids_json" \ --argjson restoredPodUids "$rollback_proxy_uids_json" \ @@ -1716,6 +1945,10 @@ jq -n \ --argjson recoverySeconds "$recovery_seconds" \ --argjson abruptEndpointWithdrawalSeconds "$abrupt_endpoint_withdrawal_seconds" \ --argjson abruptRecoverySeconds "$abrupt_recovery_seconds" \ + --argjson automaticNodeDetectionSeconds "$automatic_node_detection_seconds" \ + --argjson automaticEndpointWithdrawalSeconds "$automatic_endpoint_withdrawal_seconds" \ + --argjson automaticPodEvictionSeconds "$automatic_pod_eviction_seconds" \ + --argjson automaticRecoverySeconds "$automatic_recovery_seconds" \ --argjson baselineDistribution "$baseline_distribution_json" \ --argjson postRolloutDistribution "$post_rollout_distribution_delta_json" \ --argjson postRollbackDistribution "$post_rollback_distribution_delta_json" \ @@ -1724,7 +1957,8 @@ jq -n \ --argjson apiKeyRotation "$api_key_rotation_json" \ --argjson recoveredDistribution "$recovered_distribution_delta_json" \ --argjson abruptRecoveredDistribution "$abrupt_recovered_distribution_delta_json" \ - '{schemaVersion: 6, result: "pass", evidenceBoundary: "disposable loopback kind metadata-only content-distinct image rollout and baseline rollback, versioned immutable inbound-server TLS Secret rotation and identity rollback, bounded two-key API credential overlap/commit/rollback, planned worker loss, and operator-remediated abrupt worker-container loss; not dynamic Secret reload, not an ingress-controller, and not automatic infrastructure-failure detection, application-layer release compatibility, registry/source binding, external certificate-authority, client trust-distribution, external secret-manager, or deployment-capacity proof", + --argjson automaticRecoveredDistribution "$automatic_recovered_distribution_delta_json" \ + '{schemaVersion: 7, result: "pass", evidenceBoundary: "disposable loopback kind metadata-only content-distinct image rollout and baseline rollback, versioned immutable inbound-server TLS Secret rotation and identity rollback, bounded two-key API credential overlap/commit/rollback, planned worker loss, operator-remediated abrupt worker-container loss, and test-configured automatic node-loss detection/endpoint withdrawal/pod eviction/recovery; not dynamic Secret reload, not an ingress-controller, and not application-layer release compatibility, registry/source binding, external certificate-authority, client trust-distribution, external secret-manager, deployment-equivalent infrastructure-failure timing, or deployment-capacity proof", profileId: $profileId, repositoryRevision: $sourceRevision, images: {identityType: "local Docker content-addressed image ID", baseline: {contentId: $proxyImageId}, @@ -1736,7 +1970,8 @@ jq -n \ postApiKeyOverlapProxyReplicas: 2, postApiKeyCommitProxyReplicas: 2, postApiKeyRollbackOverlapProxyReplicas: 2, postApiKeyRollbackCommitProxyReplicas: 2, degradedProxyReplicas: 1, recoveredProxyReplicas: 2, - abruptDegradedProxyReplicas: 1, abruptRecoveredProxyReplicas: 2}, + abruptDegradedProxyReplicas: 1, abruptRecoveredProxyReplicas: 2, + automaticDegradedProxyReplicas: 1, automaticRecoveredProxyReplicas: 2}, traffic: {bothProxyReplicasServed: true, baseline: $baselineDistribution, rollout: "pass", postRollout: $postRolloutDistribution, rollback: "pass", postRollback: $postRollbackDistribution, @@ -1746,7 +1981,9 @@ jq -n \ apiKeyRollbackOverlap: "pass", apiKeyRollbackCommit: "pass", drainTransition: "pass", degraded: "pass", recovered: $recoveredDistribution, abruptTransition: "pass", abruptDegraded: "pass", - abruptRecovered: $abruptRecoveredDistribution}, + abruptRecovered: $abruptRecoveredDistribution, + automaticTransition: "pass", automaticDegraded: "pass", + automaticRecovered: $automaticRecoveredDistribution}, rolloutExercise: {triggerAnnotation: $rolloutToken, contentDistinctRuntimeImageId: true, fromRuntimeImageIds: $baselineRuntimeImageIds, toRuntimeImageIds: $candidateRuntimeImageIds, @@ -1796,7 +2033,18 @@ jq -n \ apiForcedPodNames: $abruptForcedPodNames, failedProxyPodUid: $abruptFailedProxyUid, retainedFailedProxyPodUids: 0, endpointWithdrawalSeconds: $abruptEndpointWithdrawalSeconds, - recoverySeconds: $abruptRecoverySeconds}}}' \ + recoverySeconds: $abruptRecoverySeconds}, + automatic: {stoppedWithoutDrain: $automaticWorker, + postFailureRemediationMutations: 0, + controllerNodeMonitorPeriodSeconds: 2, controllerNodeMonitorGracePeriodSeconds: 20, + unreachableTolerationSeconds: 10, detectedReadyStatus: $automaticNodeReadyStatus, + sourcePodNames: $automaticSourcePodNames, + sourcePodEvictions: $automaticSourcePodEvictions, + failedProxyPodUid: $automaticFailedProxyUid, retainedFailedProxyPodUids: 0, + nodeDetectionSeconds: $automaticNodeDetectionSeconds, + endpointWithdrawalSeconds: $automaticEndpointWithdrawalSeconds, + podEvictionSeconds: $automaticPodEvictionSeconds, + recoverySeconds: $automaticRecoverySeconds}}}' \ > "$output_dir/summary.json" for secret_file in "$api_key_file" "$candidate_api_key_file"; do @@ -1806,4 +2054,4 @@ for secret_file in "$api_key_file" "$candidate_api_key_file"; do fi done -printf 'Kubernetes two-zone image rollout/rollback, immutable TLS identity rotation/rollback, bounded API-key rotation/rollback, planned-loss, and abrupt-loss proof passed; evidence: %s\n' "$output_dir" +printf 'Kubernetes two-zone image rollout/rollback, immutable TLS identity rotation/rollback, bounded API-key rotation/rollback, planned-loss, operator-remediated abrupt-loss, and automatic worker-loss proof passed; evidence: %s\n' "$output_dir" diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 68500323..6e67ef7a 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -54,6 +54,9 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "LBP_RETRY_MAX_ATTEMPTS: \"3\"", "LBP_RETRY_BUDGET_PERCENT: \"100\"", "LBP_RETRY_NON_IDEMPOTENT: \"false\"", + "key: node.kubernetes.io/not-ready", + "key: node.kubernetes.io/unreachable", + "tolerationSeconds: 10", "secretName: loadbalancerpro-server-tls-a")) { assertTrue(workload.contains(invariant), "missing restricted workload invariant: " + invariant); } @@ -84,6 +87,10 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { assertEquals(2, count(cluster, " - role: worker")); assertTrue(cluster.contains("listenAddress: 127.0.0.1")); assertTrue(cluster.contains("hostPort: 18460")); + assertTrue(cluster.contains("apiVersion: kubeadm.k8s.io/v1beta3")); + assertTrue(cluster.contains("kind: ClusterConfiguration")); + assertTrue(cluster.contains("node-monitor-period: 2s")); + assertTrue(cluster.contains("node-monitor-grace-period: 20s")); assertTrue(cluster.contains("kind: KubeProxyConfiguration")); assertTrue(cluster.contains("mode: iptables")); assertTrue(cluster.contains("minSyncPeriod: 0s")); @@ -93,13 +100,15 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { "kindest/node:v1.34.3@sha256:08497ee19eace7b4b5348db5c6a1591d7752b164530a36f855cb0f2bdcbadd48")); JsonNode profile = new ObjectMapper().readTree(read(PROFILE)); - assertEquals(6, profile.path("schemaVersion").asInt()); + assertEquals(7, profile.path("schemaVersion").asInt()); assertEquals("example", profile.path("review").path("status").asText()); assertEquals("v1.34.3", profile.path("cluster").path("kubectlVersion").asText()); assertEquals(2, profile.path("cluster").path("workers").asInt()); assertEquals(2, profile.path("cluster").path("zones").asInt()); assertEquals("lbp-kubernetes-smoke", profile.path("cluster").path("namespace").asText()); assertEquals(30443, profile.path("cluster").path("nodePort").asInt()); + assertEquals(20, profile.path("cluster").path("nodeMonitorGracePeriodSeconds").asInt()); + assertEquals(10, profile.path("cluster").path("unreachableTolerationSeconds").asInt()); assertEquals("close-per-request", profile.path("workload").path("connectionMode").asText()); assertTrue(profile.path("workload").path("transitionSeconds").asInt() >= 40); assertTrue(profile.path("workload").path("rolloutSeconds").asInt() @@ -140,10 +149,21 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { assertTrue(profile.path("objectives").path("minimumAbruptRecoveredSuccessRatio").asDouble() >= 0.95); assertTrue(profile.path("objectives").path("maximumAbruptTransitionP99Millis").asInt() <= 6000); assertTrue(profile.path("objectives").path("maximumAbruptDegradedP99Millis").asInt() <= 6000); + assertTrue(profile.path("workload").path("automaticTransitionSeconds").asInt() + >= profile.path("objectives").path("maximumAutomaticEndpointWithdrawalSeconds").asInt() + 5); + assertTrue(profile.path("objectives").path("minimumAutomaticTransitionSuccessRatio").asDouble() >= 0.80); + assertEquals(0.95, profile.path("objectives").path("minimumAutomaticDegradedSuccessRatio").asDouble()); + assertTrue(profile.path("objectives").path("minimumAutomaticRecoveredSuccessRatio").asDouble() >= 0.95); + assertTrue(profile.path("objectives").path("maximumAutomaticTransitionP99Millis").asInt() <= 6000); + assertTrue(profile.path("objectives").path("maximumAutomaticDegradedP99Millis").asInt() <= 6000); + assertTrue(profile.path("objectives").path("maximumAutomaticNodeDetectionSeconds").asInt() + < profile.path("objectives").path("maximumAutomaticEndpointWithdrawalSeconds").asInt()); + assertTrue(profile.path("objectives").path("maximumAutomaticEndpointWithdrawalSeconds").asInt() + < profile.path("objectives").path("maximumAutomaticPodEvictionSeconds").asInt()); } @Test - void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() throws IOException { + void runnerExecutesLiveTransitionsAndManualAndAutomaticWorkerLossChecks() throws IOException { String runner = read(RUNNER); assertTrue(read(CANDIDATE).contains("metadata-only-local-candidate")); for (String behavior : List.of( @@ -154,6 +174,11 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "Refusing to reuse or delete an existing kind cluster", "Live kube-proxy config is missing", "kube-proxy-config.yaml", + "--node-monitor-period=2s", + "--node-monitor-grace-period=20s", + "kube-controller-manager-command.txt", + "node-failure-policy.json", + "Qualification Deployments do not share the bounded unreachable-node eviction policy", "Proxy replicas were not placed in distinct zones", "minDomains: 2", "-keepalive=false", @@ -250,10 +275,31 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "bothRecoveredProxyReplicasServed: true", "stoppedWithoutDrain: $abruptWorker", "retainedFailedProxyPodUids: 0", - "bothProxyReplicasServed: true")) { + "bothProxyReplicasServed: true", + "automatic-node-detection", + "docker kill \"$automatic_node\"", + "wait_for_node_unavailable", + "automatic-loss source pods still active in the API", + "automatic-source-pod-evictions.json", + "automatic-failure-timings.json", + "evictionObserved", + "Automatic worker-loss recovery retained the failed worker pod UID", + "automatic-recovered-distribution-delta.json", + "postFailureRemediationMutations: 0", + "controllerNodeMonitorGracePeriodSeconds: 20", + "deployment-equivalent infrastructure-failure timing")) { assertTrue(runner.contains(behavior), "missing live Kubernetes proof behavior: " + behavior); } - assertTrue(read(CONTRACT).contains("rejected 65 unsafe profiles without creating a cluster")); + String automaticFailureWindow = runner.substring( + runner.indexOf("automatic_failure_started_epoch="), + runner.indexOf("automatic_recovery_started_epoch=")); + for (String mutatingCommand : List.of( + "kubectl apply", "kubectl cordon", "kubectl delete", "kubectl drain", "kubectl patch", "kubectl taint")) { + assertFalse( + automaticFailureWindow.contains(mutatingCommand), + "automatic worker-loss remediation must not mutate Kubernetes state with " + mutatingCommand); + } + assertTrue(read(CONTRACT).contains("rejected 83 unsafe profiles without creating a cluster")); assertFalse(runner.contains("--insecure")); assertFalse(runner.contains("--validate=false")); }