From 929fa131af42c916334017c765a6527bb8c65b82 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 21:19:17 +0000 Subject: [PATCH 01/45] feat: add managed agent binary upgrades --- cmd/aks-flex-node/main.go | 1 + docs/usages/operations.md | 27 + hack/controller-deployment/rbac.yaml | 30 + hack/e2e/README.md | 26 +- hack/e2e/lib/agent-upgrade.sh | 243 ++++++++ hack/e2e/lib/controller.sh | 9 +- hack/e2e/run.sh | 15 +- pkg/cmd/daemon/agent_upgrade_recovery.go | 29 + pkg/daemon/agent_upgrade.go | 577 ++++++++++++++++++ pkg/daemon/agent_upgrade_binary.go | 456 ++++++++++++++ pkg/daemon/agent_upgrade_binary_test.go | 334 ++++++++++ pkg/daemon/agent_upgrade_test.go | 390 ++++++++++++ .../aks-flex-node-agent-recovery.service | 6 + pkg/daemon/assets/aks-flex-node-agent.service | 3 +- pkg/daemon/assets/aks-flex-node-recovery.sh | 19 + pkg/daemon/daemon.go | 24 + pkg/daemon/lifecycle.go | 43 +- pkg/daemon/lifecycle_test.go | 30 + pkg/daemon/machineoperation_reconciler.go | 80 ++- .../machineoperation_reconciler_test.go | 168 ++++- pkg/utils/utilexec/exec.go | 7 + 21 files changed, 2497 insertions(+), 20 deletions(-) create mode 100644 hack/e2e/lib/agent-upgrade.sh create mode 100644 pkg/cmd/daemon/agent_upgrade_recovery.go create mode 100644 pkg/daemon/agent_upgrade.go create mode 100644 pkg/daemon/agent_upgrade_binary.go create mode 100644 pkg/daemon/agent_upgrade_binary_test.go create mode 100644 pkg/daemon/agent_upgrade_test.go create mode 100644 pkg/daemon/assets/aks-flex-node-agent-recovery.service create mode 100644 pkg/daemon/assets/aks-flex-node-recovery.sh create mode 100644 pkg/daemon/lifecycle_test.go diff --git a/cmd/aks-flex-node/main.go b/cmd/aks-flex-node/main.go index 5e80a8e1..eb071ee9 100644 --- a/cmd/aks-flex-node/main.go +++ b/cmd/aks-flex-node/main.go @@ -30,6 +30,7 @@ func main() { rootCmd.AddCommand(bootstrapdata.NewCommand()) rootCmd.AddCommand(preflight.NewCommand()) rootCmd.AddCommand(daemon.NewCommand()) + rootCmd.AddCommand(daemon.NewAgentUpgradeRecoveryCommand()) rootCmd.AddCommand(reset.NewCommand()) rootCmd.AddCommand(version.NewCommand()) rootCmd.AddCommand(token.Command) diff --git a/docs/usages/operations.md b/docs/usages/operations.md index fae735a4..26057c20 100644 --- a/docs/usages/operations.md +++ b/docs/usages/operations.md @@ -47,6 +47,33 @@ systemctl is-active aks-flex-node-agent journalctl -u aks-flex-node-agent -f ``` +## Managed Agent Upgrade + +When the Machina `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTPS release archive and the SHA-256 of the compressed archive: + +```yaml +apiVersion: unbounded-cloud.io/v1alpha3 +kind: MachineOperation +metadata: + name: upgrade-agent-worker-01 +spec: + machineRef: worker-01 + operationKind: AgentUpgrade + parameters: + downloadURL: https://example.com/aks-flex-node-linux-amd64.tar.gz + sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +``` + +The archive must contain exactly the architecture-specific release member used by AKS Flex Node (`aks-flex-node-linux-amd64` or `aks-flex-node-linux-arm64`). The daemon verifies the archive digest and candidate `version` command before switching its blue/green binary links. It also atomically updates the binary in the active nspawn rootfs so kubelet exec authentication uses the same version. + +The restarted daemon marks the operation `Complete`. If the candidate cannot remain running, systemd restores the last-known-good host and nspawn binaries and marks the operation `Failed`. URL query strings, which may contain SAS credentials, are omitted from logs and operation status. + +MachineOperations are cluster-scoped. The daemon group requires cluster-wide read access to MachineOperations and Nodes, plus MachineOperation status update access, so restrict who can create operations and treat parameter values as sensitive API data. Prefer short-lived, read-only download credentials. + +```bash +kubectl get machineoperation upgrade-agent-worker-01 -w +``` + ## Nspawn Worker Inspect the local nspawn-backed worker: diff --git a/hack/controller-deployment/rbac.yaml b/hack/controller-deployment/rbac.yaml index bea3b066..acf0ed58 100644 --- a/hack/controller-deployment/rbac.yaml +++ b/hack/controller-deployment/rbac.yaml @@ -103,3 +103,33 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: aks-flex-node-daemons +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aks-flex-node-daemon-machineoperations +rules: + - apiGroups: ["unbounded-cloud.io"] + resources: ["machineoperations"] + verbs: ["get", "list", "watch"] + - apiGroups: ["unbounded-cloud.io"] + resources: ["machineoperations/status"] + verbs: ["get", "patch", "update"] + # The shared MachineOperation selector implementation evaluates labels on the + # local Node when an operation uses machineSelector. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aks-flex-node-daemon-machineoperations +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aks-flex-node-daemon-machineoperations +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: aks-flex-node-daemons diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 20496e4f..b1238c1d 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -53,8 +53,9 @@ The default `all` command runs: 6. Validate node readiness, node-problem-detector status, and run smoke workloads. 7. Unjoin all Flex Nodes and verify they are absent, including reset cleanup of host network artifacts. 8. Rejoin all Flex Nodes and validate again. -9. Run controller-machine-driven repave validation. -10. Collect logs and clean up Azure resources. +9. Validate managed agent upgrade, forced rollback, retry, nspawn synchronization, and kubelet authentication. +10. Run controller-machine-driven repave validation after the agent upgrade. +11. Collect logs and clean up Azure resources. ## Commands @@ -77,6 +78,7 @@ The default `all` command runs: | `validate` | Verify joined nodes, node-problem-detector status, and run smoke tests. | | `validate-absent` | Verify Flex Node objects are absent after unjoin. | | `smoke` | Run smoke workloads only. | +| `agent-upgrade` | Validate managed agent upgrade, forced rollback, retry, and nspawn synchronization. | | `upgrade-drift` | Validate controller-machine-driven repave to the alternate nspawn side. | | `logs` | Collect logs from VMs. | | `cleanup` | Collect logs and delete Azure resources. | @@ -121,6 +123,7 @@ Additional environment variables: | `E2E_SSH_WAIT_TIMEOUT` | `300` | Timeout in seconds while waiting for SSH. | | `E2E_NODE_JOIN_TIMEOUT` | `300` | Timeout in seconds while waiting for node bootstrap. | | `E2E_POD_READY_TIMEOUT` | `120` | Timeout in seconds while waiting for smoke pods. | +| `E2E_AGENT_UPGRADE_TIMEOUT` | `300` | Timeout in seconds while waiting for an AgentUpgrade result. | | `E2E_DRIFT_UPGRADE_TIMEOUT` | `900` | Timeout in seconds while waiting for repave. | | `AZURE_SUBSCRIPTION_ID` | auto-detected | Azure subscription. | | `AZURE_TENANT_ID` | auto-detected | Azure tenant. | @@ -143,6 +146,24 @@ cluster under the lowercase VM name. Each join path uploads the locally built binary, renders a config file, installs the binary through `scripts/install.sh` with `AKS_FLEX_NODE_LOCAL_BINARY`, and starts the node through a transient systemd unit. The installed agent service is then validated with systemd checks. +## Agent Upgrade Validation + +The `agent-upgrade` command uses the bootstrap-token VM to exercise the complete managed binary lifecycle: + +1. Serve architecture-specific release archives over trusted loopback HTTPS. +2. Submit an `AgentUpgrade` with an archive SHA-256 and a query credential. +3. Verify successful daemon restart, operation completion, binary replacement, and host/nspawn binary equality. +4. Restart kubelet to exercise the synchronized nspawn exec-credential binary and require the Node to remain Ready. +5. Upgrade to a candidate that passes `version` but fails daemon startup, then verify automatic rollback and a failed operation. +6. Confirm status does not expose the sensitive URL query and retry successfully into the inactive slot. +7. Run a workload before the subsequent repave test. + +Run it against an already joined environment: + +```bash +./hack/e2e/run.sh agent-upgrade +``` + ## Repave Validation The `upgrade-drift` command validates the controller-machine-driven repave path: @@ -215,6 +236,7 @@ hack/e2e/ node-join-token.sh Bootstrap token join/unjoin. node-join-offline.sh Offline artifacts join/unjoin. node-join-kubeadm.sh Kubeadm-style bootstrap-token join/unjoin. + agent-upgrade.sh Managed agent upgrade and rollback validation. upgrade-drift.sh Controller machine goal repave validation. validate.sh Node readiness and smoke tests. cleanup.sh Log collection and Azure resource cleanup. diff --git a/hack/e2e/lib/agent-upgrade.sh b/hack/e2e/lib/agent-upgrade.sh new file mode 100644 index 00000000..27e83256 --- /dev/null +++ b/hack/e2e/lib/agent-upgrade.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +# ============================================================================= +# AgentUpgrade blue/green, rollback, retry, and nspawn synchronization E2E test. +# ============================================================================= +set -euo pipefail + +[[ -n "${_E2E_AGENT_UPGRADE_LOADED:-}" ]] && return 0 +readonly _E2E_AGENT_UPGRADE_LOADED=1 + +# shellcheck disable=SC1091 +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +_agent_upgrade_ensure_api() { + local unbounded_dir + unbounded_dir="$(cd "${REPO_ROOT}" && go list -m -f '{{.Dir}}' github.com/Azure/unbounded)" + kubectl apply -f "${unbounded_dir}/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml" + kubectl wait --for=condition=Established \ + customresourcedefinition/machineoperations.unbounded-cloud.io --timeout=60s +} + +_agent_upgrade_prepare_server() { + local vm_ip="$1" + local upload_path="/tmp/aks-flex-node-e2e-upgrade-binary" + remote_copy "${E2E_BINARY}" "${vm_ip}" "${upload_path}" + + remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' +set -euo pipefail +work=/opt/aks-flex-node-e2e-upgrade +sudo rm -rf "${work}" +sudo install -d -m 0755 "${work}" +sudo install -m 0755 /tmp/aks-flex-node-e2e-upgrade-binary "${work}/aks-flex-node-linux-amd64" +sudo tar -C "${work}" -czf "${work}/success.tar.gz" aks-flex-node-linux-amd64 + +cat >/tmp/aks-flex-node-e2e-broken <<'BROKEN' +#!/bin/sh +if [ "${1:-}" = "version" ]; then + echo "e2e-forced-daemon-failure" + exit 0 +fi +exit 42 +BROKEN +sudo install -m 0755 /tmp/aks-flex-node-e2e-broken "${work}/aks-flex-node-linux-amd64" +sudo tar -C "${work}" -czf "${work}/failure.tar.gz" aks-flex-node-linux-amd64 +check_dir="$(mktemp -d)" +sudo tar -C "${check_dir}" -xzf "${work}/failure.tar.gz" +sudo "${check_dir}/aks-flex-node-linux-amd64" version >/dev/null +if sudo "${check_dir}/aks-flex-node-linux-amd64" agent >/dev/null 2>&1; then + echo 'forced-failure archive unexpectedly starts the daemon command' >&2 + exit 1 +fi +rm -rf "${check_dir}" +sudo install -m 0755 /tmp/aks-flex-node-e2e-upgrade-binary "${work}/aks-flex-node-linux-amd64" + +sudo openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj '/CN=127.0.0.1' \ + -addext 'subjectAltName=IP:127.0.0.1' \ + -keyout "${work}/server.key" -out "${work}/server.crt" >/dev/null 2>&1 +sudo cp "${work}/server.crt" /usr/local/share/ca-certificates/aks-flex-node-e2e-upgrade.crt +sudo update-ca-certificates >/dev/null +# Reload Go's system root pool in the long-running daemon after adding the +# short-lived test CA. +sudo systemctl restart aks-flex-node-agent.service +cat >/tmp/aks-flex-node-e2e-upgrade-server.py <<'PY' +import http.server +import ssl + +server = http.server.ThreadingHTTPServer(("127.0.0.1", 18443), http.server.SimpleHTTPRequestHandler) +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain("/opt/aks-flex-node-e2e-upgrade/server.crt", "/opt/aks-flex-node-e2e-upgrade/server.key") +server.socket = context.wrap_socket(server.socket, server_side=True) +server.serve_forever() +PY +sudo install -m 0755 /tmp/aks-flex-node-e2e-upgrade-server.py "${work}/server.py" +sudo systemctl stop aks-flex-node-e2e-upgrade-server.service 2>/dev/null || true +sudo systemd-run --unit=aks-flex-node-e2e-upgrade-server.service \ + --property=WorkingDirectory="${work}" \ + /usr/bin/python3 "${work}/server.py" >/dev/null +for _ in $(seq 1 30); do + if curl --silent --fail https://127.0.0.1:18443/success.tar.gz >/dev/null; then + exit 0 + fi + sleep 1 +done +echo 'upgrade archive server did not become ready' >&2 +exit 1 +REMOTE +} + +_agent_upgrade_digest() { + local vm_ip="$1" archive="$2" + remote_exec "${vm_ip}" "sha256sum /opt/aks-flex-node-e2e-upgrade/${archive} | awk '{print \$1}'" +} + +_agent_upgrade_apply() { + local operation="$1" vm_name="$2" archive="$3" digest="$4" token="$5" + cat </dev/null || true)" + ready="$(kubectl get node "${vm_name}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + if [[ -n "${renew}" && "${renew}" != "${before_renew}" && "${ready}" == "True" ]]; then + log_success "Kubelet renewed its lease after restart using the synchronized exec credential binary" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + log_error "Kubelet did not renew its lease after AgentUpgrade" + return 1 +} + +agent_upgrade_e2e() { + log_section "Managed AgentUpgrade E2E" + local vm_name vm_ip suffix success_digest failure_digest before before_binary_digest success_snapshot success_binary_digest rollback_snapshot rollback_binary_digest retry_snapshot retry_binary_digest + vm_name="$(state_get token_vm_name)" + vm_ip="$(state_get token_vm_ip)" + suffix="$(date +%s)" + + validate_node_joined "${vm_name}" + _agent_upgrade_ensure_api + _agent_upgrade_prepare_server "${vm_ip}" + success_digest="$(_agent_upgrade_digest "${vm_ip}" success.tar.gz)" + failure_digest="$(_agent_upgrade_digest "${vm_ip}" failure.tar.gz)" + before="$(_agent_upgrade_snapshot "${vm_ip}")" + before_binary_digest="$(cut -d'|' -f3 <<<"${before}")" + log_info "Pre-upgrade agent snapshot: ${before}" + + local success_op="agent-upgrade-success-${suffix}" + _agent_upgrade_apply "${success_op}" "${vm_name}" success.tar.gz "${success_digest}" "success-${suffix}" + _agent_upgrade_wait_phase "${success_op}" Complete + validate_node_joined "${vm_name}" + _agent_upgrade_assert_synchronized "${vm_ip}" + success_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" + IFS='|' read -r _ _ success_binary_digest _ <<<"${success_snapshot}" + if [[ -z "${success_binary_digest}" || "${success_binary_digest}" == "${before_binary_digest}" ]]; then + log_error "Successful AgentUpgrade did not replace the running binary: before=${before} after=${success_snapshot}" + return 1 + fi + + # Restart kubelet and require a fresh Lease renewal so this proves the + # synchronized nspawn exec-credential binary can still authenticate. + _agent_upgrade_validate_kubelet_auth "${vm_name}" "${vm_ip}" + + local failure_op="agent-upgrade-rollback-${suffix}" + _agent_upgrade_apply "${failure_op}" "${vm_name}" failure.tar.gz "${failure_digest}" "failure-${suffix}" + _agent_upgrade_wait_phase "${failure_op}" Failed + rollback_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" + IFS='|' read -r _ _ rollback_binary_digest _ <<<"${rollback_snapshot}" + if [[ "${rollback_binary_digest}" != "${success_binary_digest}" ]]; then + log_error "Rollback did not restore the successful candidate: success=${success_snapshot} rollback=${rollback_snapshot}" + return 1 + fi + if kubectl get machineoperation "${failure_op}" -o jsonpath='{.status.message}' | grep -q "failure-${suffix}"; then + log_error "AgentUpgrade status leaked sensitive URL query data" + return 1 + fi + remote_exec "${vm_ip}" 'sudo systemctl is-active --quiet aks-flex-node-agent.service' + _agent_upgrade_assert_synchronized "${vm_ip}" + validate_node_joined "${vm_name}" + + local retry_op="agent-upgrade-retry-${suffix}" + _agent_upgrade_apply "${retry_op}" "${vm_name}" success.tar.gz "${success_digest}" "retry-${suffix}" + _agent_upgrade_wait_phase "${retry_op}" Complete + retry_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" + IFS='|' read -r _ _ retry_binary_digest _ <<<"${retry_snapshot}" + if [[ "${retry_binary_digest}" != "${success_binary_digest}" ]]; then + log_error "Retry did not install the expected successful candidate: ${retry_snapshot}" + return 1 + fi + _agent_upgrade_assert_synchronized "${vm_ip}" + validate_node_joined "${vm_name}" + smoke_test "${vm_name}" "agent-upgrade" + + log_success "Managed AgentUpgrade success, rollback, and retry E2E passed" +} diff --git a/hack/e2e/lib/controller.sh b/hack/e2e/lib/controller.sh index 2f2046ac..7f4b0d54 100644 --- a/hack/e2e/lib/controller.sh +++ b/hack/e2e/lib/controller.sh @@ -285,7 +285,14 @@ _wait_for_controller_ready() { } _ensure_flex_controller_unlocked() { - local image + local image unbounded_dir + # Install the optional API before any Flex daemon starts so its startup-time + # discovery enables MachineOperation watches without requiring a restart. + unbounded_dir="$(cd "${REPO_ROOT}" && go list -m -f '{{.Dir}}' github.com/Azure/unbounded)" + kubectl apply -f "${unbounded_dir}/deploy/machina/crd/unbounded-cloud.io_machineoperations.yaml" || return 1 + kubectl wait --for=condition=Established \ + customresourcedefinition/machineoperations.unbounded-cloud.io --timeout=60s || return 1 + image="$(_controller_image_from_state_or_env)" if [[ -n "${E2E_CONTROLLER_IMAGE:-}" ]]; then diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index 6a41c1bf..5f418ae2 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -22,6 +22,7 @@ # validate Verify nodes joined + run smoke tests # validate-absent Verify all flex nodes are gone after unjoin # smoke Run smoke tests only (pods on flex nodes) +# agent-upgrade Validate managed binary upgrade, rollback, and retry # upgrade-drift Run controller-machine Kubernetes version drift repave test # logs Collect logs from VMs # cleanup Tear down Azure resources @@ -104,6 +105,8 @@ source "${SCRIPT_DIR}/lib/validate.sh" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/upgrade-drift.sh" # shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/agent-upgrade.sh" +# shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/cleanup.sh" # shellcheck disable=SC1091 source "${SCRIPT_DIR}/lib/runner.sh" @@ -128,7 +131,7 @@ usage() { parse_args() { while [[ $# -gt 0 ]]; do case "$1" in - all|infra|join|join-msi|join-token|join-offline|join-kubeadm|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|validate|validate-absent|smoke|upgrade-drift|logs|cleanup|runner-cleanup|status) + all|infra|join|join-msi|join-token|join-offline|join-kubeadm|unjoin|unjoin-msi|unjoin-token|unjoin-offline|unjoin-kubeadm|validate|validate-absent|smoke|agent-upgrade|upgrade-drift|logs|cleanup|runner-cleanup|status) COMMAND="$1"; shift ;; -g|--resource-group) export E2E_RESOURCE_GROUP="$2"; shift 2 ;; -l|--location) export E2E_LOCATION="$2"; shift 2 ;; @@ -197,7 +200,10 @@ cmd_all() { validate_all_nodes smoke_test_all || exit_code=1 - # ── Controller-backed machine repave ─────────────────────────────────── + # ── Managed host agent binary upgrade ───────────────────────────────── + agent_upgrade_e2e + + # ── Controller-backed machine repave after agent upgrade ─────────────── upgrade_drift_all # Collect logs (always, even if tests fail) @@ -310,6 +316,11 @@ main() { smoke) smoke_test_all ;; + agent-upgrade) + ensure_binary + ensure_cluster_dependencies + agent_upgrade_e2e + ;; upgrade-drift) ensure_binary ensure_cluster_dependencies diff --git a/pkg/cmd/daemon/agent_upgrade_recovery.go b/pkg/cmd/daemon/agent_upgrade_recovery.go new file mode 100644 index 00000000..8c6389cc --- /dev/null +++ b/pkg/cmd/daemon/agent_upgrade_recovery.go @@ -0,0 +1,29 @@ +package daemon + +import ( + "fmt" + + "github.com/spf13/cobra" + + hostdaemon "github.com/Azure/AKSFlexNode/pkg/daemon" +) + +// NewAgentUpgradeRecoveryCommand returns the internal command used by the +// systemd recovery unit. It remains callable only as a local root operation. +func NewAgentUpgradeRecoveryCommand() *cobra.Command { + var message string + cmd := &cobra.Command{ + Use: "recover-agent-upgrade", + Short: "Restore the last-known-good agent after a failed upgrade", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := hostdaemon.RecoverAgentUpgrade(cmd.Context(), message); err != nil { + return fmt.Errorf("recover AgentUpgrade: %w", err) + } + return nil + }, + } + cmd.Flags().StringVar(&message, "message", "", "failure message to publish") + return cmd +} diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go new file mode 100644 index 00000000..f1410cf9 --- /dev/null +++ b/pkg/daemon/agent_upgrade.go @@ -0,0 +1,577 @@ +package daemon + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/AKSFlexNode/pkg/utils/utilexec" + "github.com/Azure/AKSFlexNode/pkg/utils/utilio" + machinav1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + agentdaemon "github.com/Azure/unbounded/pkg/agent/daemon" + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +const ( + agentUpgradeDownloadURLParameter = "downloadURL" + agentUpgradeSHA256Parameter = "sha256" +) + +var errAgentUpgradeAlreadyPending = errors.New("AgentUpgrade operation is already pending") + +func defaultAgentUpgradePaths() agentUpgradePaths { + const binaryDir = "/usr/local/lib/aks-flex-node" + return agentUpgradePaths{ + BinaryPath: "/usr/local/bin/aks-flex-node", + BluePath: filepath.Join(binaryDir, "aks-flex-node-blue"), + GreenPath: filepath.Join(binaryDir, "aks-flex-node-green"), + CurrentPath: filepath.Join(binaryDir, "aks-flex-node-current"), + LastGoodPath: filepath.Join(binaryDir, "aks-flex-node-last-good"), + SignalPath: "/etc/aks-flex-node/agent-upgrade-signal.json", + } +} + +type agentUpgradeRequest struct { + downloadURL string + sha256 string +} + +func parseAgentUpgradeRequest(parameters map[string]string) (agentUpgradeRequest, error) { + request := agentUpgradeRequest{ + downloadURL: strings.TrimSpace(parameters[agentUpgradeDownloadURLParameter]), + sha256: strings.TrimSpace(parameters[agentUpgradeSHA256Parameter]), + } + if request.downloadURL == "" { + return agentUpgradeRequest{}, fmt.Errorf("missing required parameter %q", agentUpgradeDownloadURLParameter) + } + if _, err := validateAgentUpgradeURL(request.downloadURL); err != nil { + return agentUpgradeRequest{}, err + } + if request.sha256 == "" { + return agentUpgradeRequest{}, fmt.Errorf("missing required parameter %q", agentUpgradeSHA256Parameter) + } + if _, err := parseAgentUpgradeSHA256(request.sha256); err != nil { + return agentUpgradeRequest{}, err + } + return request, nil +} + +type agentUpgradeSignal struct { + OperationName string `json:"operationName"` + ActiveMachine string `json:"activeMachine,omitempty"` + CandidatePath string `json:"candidatePath,omitempty"` + InitiatingDaemonInstance string `json:"initiatingDaemonInstance,omitempty"` + RecoveryRequired bool `json:"recoveryRequired,omitempty"` + Failure string `json:"failure,omitempty"` +} + +type agentUpgradeSignalStore struct { + path string +} + +func (s agentUpgradeSignalStore) recordPending(operationName, activeMachine, daemonInstance string) error { + return s.write(agentUpgradeSignal{ + OperationName: operationName, + ActiveMachine: activeMachine, + InitiatingDaemonInstance: daemonInstance, + }) +} + +func (s agentUpgradeSignalStore) recordCandidate(candidatePath string) error { + signal, err := s.read() + if err != nil { + return err + } + if signal == nil { + return fmt.Errorf("no pending AgentUpgrade signal") + } + signal.CandidatePath = candidatePath + return s.write(*signal) +} + +func (s agentUpgradeSignalStore) recordFailure(message string) error { + signal, err := s.read() + if err != nil { + return err + } + if signal == nil { + return nil + } + message = strings.TrimSpace(message) + if message == "" { + message = "upgraded daemon failed to start; restored last-good binary" + } + signal.Failure = message + signal.RecoveryRequired = true + return s.write(*signal) +} + +func (s agentUpgradeSignalStore) read() (*agentUpgradeSignal, error) { + data, err := os.ReadFile(s.path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read AgentUpgrade signal: %w", err) + } + var signal agentUpgradeSignal + if err := json.Unmarshal(data, &signal); err != nil { + return nil, fmt.Errorf("decode AgentUpgrade signal: %w", err) + } + signal.OperationName = strings.TrimSpace(signal.OperationName) + signal.ActiveMachine = strings.TrimSpace(signal.ActiveMachine) + signal.CandidatePath = strings.TrimSpace(signal.CandidatePath) + signal.Failure = strings.TrimSpace(signal.Failure) + if signal.OperationName == "" { + return nil, fmt.Errorf("AgentUpgrade signal has no operation name") + } + if signal.ActiveMachine != "" && !validNspawnMachine(signal.ActiveMachine) { + return nil, fmt.Errorf("AgentUpgrade signal has invalid active machine %q", signal.ActiveMachine) + } + return &signal, nil +} + +func (s agentUpgradeSignalStore) write(signal agentUpgradeSignal) error { + data, err := json.Marshal(signal) + if err != nil { + return fmt.Errorf("encode AgentUpgrade signal: %w", err) + } + if err := utilio.WriteFile(s.path, append(data, '\n'), 0o600); err != nil { + return fmt.Errorf("write AgentUpgrade signal: %w", err) + } + return nil +} + +func (s agentUpgradeSignalStore) clear() error { + if err := os.Remove(s.path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove AgentUpgrade signal: %w", err) + } + return nil +} + +type agentUpgradeExecutor interface { + RecordPending(context.Context, string) error + RecordFailure(string) error + Stage(context.Context, agentUpgradeRequest) error + Abort(context.Context) error + Restart(context.Context) error +} + +type agentUpgradeStateLoader interface { + LoadState(context.Context) (*State, error) +} + +type hostAgentUpgradeExecutor struct { + log *slog.Logger + paths agentUpgradePaths + state agentUpgradeStateLoader + signals agentUpgradeSignalStore + runSystemdRun func(context.Context, ...string) error + finishMachineOperation func(context.Context, client.Client, agentdaemon.MachineOperation, agentdaemon.MachineOperationResult[int64]) error + runningExecutable func() (string, error) + instanceID string +} + +func newHostAgentUpgradeExecutor(log *slog.Logger, state agentUpgradeStateLoader) (*hostAgentUpgradeExecutor, error) { + paths := defaultAgentUpgradePaths() + instanceID, err := newDaemonInstanceID() + if err != nil { + return nil, fmt.Errorf("create daemon instance ID: %w", err) + } + return &hostAgentUpgradeExecutor{ + log: log, + paths: paths, + state: state, + signals: agentUpgradeSignalStore{path: paths.SignalPath}, + runSystemdRun: func(ctx context.Context, args ...string) error { + return utilexec.RunCmd(ctx, log, utilexec.SystemdRun(), args...) + }, + finishMachineOperation: agentdaemon.FinishMachineOperation, + runningExecutable: runningAgentExecutable, + instanceID: instanceID, + }, nil +} + +func newDaemonInstanceID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + return hex.EncodeToString(value[:]), nil +} + +func (e *hostAgentUpgradeExecutor) RecordPending(ctx context.Context, operationName string) error { + existing, err := e.signals.read() + if err != nil { + return err + } + if existing != nil { + if existing.OperationName == operationName { + return errAgentUpgradeAlreadyPending + } + return fmt.Errorf("another AgentUpgrade operation %q is pending", existing.OperationName) + } + state, err := e.state.LoadState(ctx) + if err != nil { + return fmt.Errorf("load daemon state for AgentUpgrade: %w", err) + } + if state == nil || !validNspawnMachine(state.ActiveMachine) { + return fmt.Errorf("no valid active nspawn machine for AgentUpgrade") + } + if err := e.signals.recordPending(operationName, state.ActiveMachine, e.instanceID); err != nil { + return err + } + return nil +} + +func (e *hostAgentUpgradeExecutor) RecordFailure(message string) error { + return e.signals.recordFailure(message) +} + +func (e *hostAgentUpgradeExecutor) Stage(ctx context.Context, request agentUpgradeRequest) error { + if err := ensureAgentUpgradeLayout(ctx, e.log, e.paths); err != nil { + return fmt.Errorf("initialize agent binary layout: %w", err) + } + current, err := resolvedExecutable(e.paths.CurrentPath) + if err != nil { + return fmt.Errorf("resolve current agent binary: %w", err) + } + candidate := e.paths.BluePath + if current == e.paths.BluePath { + candidate = e.paths.GreenPath + } + if err := e.signals.recordCandidate(candidate); err != nil { + return err + } + if err := installAndSwitchAgentBinary(ctx, e.log, request.downloadURL, request.sha256, e.paths); err != nil { + return err + } + + signal, err := e.signals.read() + if err != nil { + return e.rollbackAfterStage(ctx, err) + } + if signal == nil || !validNspawnMachine(signal.ActiveMachine) { + return e.rollbackAfterStage(ctx, fmt.Errorf("pending AgentUpgrade signal has no valid active machine")) + } + current, err = resolvedExecutable(e.paths.CurrentPath) + if err != nil { + return e.rollbackAfterStage(ctx, fmt.Errorf("resolve staged agent binary: %w", err)) + } + if current != candidate { + return e.rollbackAfterStage(ctx, fmt.Errorf("staged agent binary resolved to unexpected slot")) + } + if err := synchronizeNspawnAgentBinary(current, signal.ActiveMachine); err != nil { + return e.rollbackAfterStage(ctx, fmt.Errorf("synchronize active nspawn agent binary: %w", err)) + } + return nil +} + +func (e *hostAgentUpgradeExecutor) rollbackAfterStage(ctx context.Context, stageErr error) error { + cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) + defer cancel() + if rollbackErr := e.rollback(cleanupCtx); rollbackErr != nil { + return fmt.Errorf("%w; rollback failed: %v", stageErr, rollbackErr) + } + return stageErr +} + +func (e *hostAgentUpgradeExecutor) Abort(ctx context.Context) error { + cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) + defer cancel() + if err := e.rollback(cleanupCtx); err != nil { + // Preserve the signal so startup recovery can retry the rollback. + return err + } + return e.signals.clear() +} + +func agentUpgradeCleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) +} + +func (e *hostAgentUpgradeExecutor) rollback(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + lastGood, err := resolvedExecutable(e.paths.LastGoodPath) + if err != nil { + return fmt.Errorf("resolve last-good agent binary: %w", err) + } + if err := replaceSymlink(e.paths.CurrentPath, lastGood); err != nil { + return fmt.Errorf("restore last-good agent binary: %w", err) + } + signal, err := e.signals.read() + if err != nil { + return err + } + if signal != nil && signal.ActiveMachine != "" { + if err := synchronizeNspawnAgentBinary(lastGood, signal.ActiveMachine); err != nil { + return fmt.Errorf("restore active nspawn agent binary: %w", err) + } + } + return nil +} + +func (e *hostAgentUpgradeExecutor) Restart(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // A direct self-restart can terminate the systemctl child before the + // handler records a successful handoff. Schedule the restart in a separate + // transient unit so this process returns while its durable signal is intact. + unit := fmt.Sprintf("aks-flex-node-agent-upgrade-restart-%d", time.Now().UnixNano()) + return e.runSystemdRun( + ctx, + "--quiet", + "--collect", + "--unit="+unit, + "--on-active=1s", + "/usr/bin/systemctl", + "restart", + ServiceUnitName, + ) +} + +func validNspawnMachine(machine string) bool { + return machine == goalstates.NSpawnMachineKube1 || machine == goalstates.NSpawnMachineKube2 +} + +func synchronizeNspawnAgentBinary(sourcePath, machine string) error { + if !validNspawnMachine(machine) { + return fmt.Errorf("invalid nspawn machine %q", machine) + } + destination := filepath.Join("/var/lib/machines", machine, "usr", "local", "bin", "aks-flex-node") + if err := copyExecutable(sourcePath, destination); err != nil { + return fmt.Errorf("copy agent binary to %s: %w", machine, err) + } + return nil +} + +// RecoverAgentUpgrade records failure and restores both host and active nspawn +// binaries. It is invoked by the systemd recovery unit through last-good. +func RecoverAgentUpgrade(ctx context.Context, message string) error { + paths := defaultAgentUpgradePaths() + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.recordFailure(message); err != nil { + return err + } + lastGood, err := resolvedExecutable(paths.LastGoodPath) + if err != nil { + return fmt.Errorf("resolve last-good agent binary: %w", err) + } + if err := replaceSymlink(paths.CurrentPath, lastGood); err != nil { + return fmt.Errorf("restore last-good agent binary: %w", err) + } + signal, err := signals.read() + if err != nil { + return err + } + if signal != nil && signal.ActiveMachine != "" { + if err := synchronizeNspawnAgentBinary(lastGood, signal.ActiveMachine); err != nil { + return err + } + } + return ctx.Err() +} + +func retryAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c client.Client, executor *hostAgentUpgradeExecutor) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := publishAndClearAgentUpgradeSignal(ctx, log, c, executor); err != nil && ctx.Err() == nil { + log.Warn("failed to publish durable AgentUpgrade result; will retry", "error", err) + } + } + } +} + +func publishAndClearAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c client.Client, executor *hostAgentUpgradeExecutor) error { + paths := executor.paths + signals := executor.signals + signal, err := signals.read() + if err != nil { + return err + } + if signal == nil { + return nil + } + if signal.Failure == "" && signal.InitiatingDaemonInstance == executor.instanceID { + // The initiating daemon must not consume its own signal while staging. + return nil + } + + if signal.Failure == "" { + if validationErr := validateStartedAgentUpgrade(paths, signal); validationErr != nil { + signal.Failure = validationErr.Error() + signal.RecoveryRequired = true + if err := signals.write(*signal); err != nil { + return err + } + } + } + + result := agentdaemon.MachineOperationResult[int64]{ + Phase: machinav1alpha3.OperationPhaseComplete, + Reason: "Succeeded", + Message: "AgentUpgrade completed", + } + if signal.Failure != "" { + result.Phase = machinav1alpha3.OperationPhaseFailed + result.Reason = "DaemonFailed" + result.Message = signal.Failure + + if err := rollbackAgentUpgradeFiles(paths, signal); err != nil { + return fmt.Errorf("roll back failed AgentUpgrade: %w", err) + } + } + + finishOperation := executor.finishMachineOperation + if finishOperation == nil { + finishOperation = agentdaemon.FinishMachineOperation + } + finishErr := finishOperation(ctx, c, agentdaemon.MachineOperation{Name: signal.OperationName}, result) + if signal.RecoveryRequired { + lastGood, err := resolvedExecutable(paths.LastGoodPath) + if err != nil { + return fmt.Errorf("resolve last-good agent for recovery restart: %w", err) + } + runningExecutable := executor.runningExecutable + if runningExecutable == nil { + runningExecutable = runningAgentExecutable + } + running, err := runningExecutable() + if err != nil { + return err + } + if running != lastGood { + cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) + restartErr := executor.Restart(cleanupCtx) + cancel() + if finishErr != nil || restartErr != nil { + return errors.Join( + wrapOptionalError("publish AgentUpgrade result", finishErr), + wrapOptionalError("restart last-good agent", restartErr), + ) + } + // Keep the signal until the last-good process confirms it is running. + return nil + } + } + if finishErr != nil { + return fmt.Errorf("publish AgentUpgrade result: %w", finishErr) + } + if err := signals.clear(); err != nil { + return err + } + log.Info("published AgentUpgrade result", "operation", signal.OperationName, "phase", result.Phase) + return nil +} + +func runningAgentExecutable() (string, error) { + path, err := os.Executable() + if err != nil { + return "", fmt.Errorf("resolve running agent executable: %w", err) + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", fmt.Errorf("resolve running agent executable symlinks: %w", err) + } + return resolved, nil +} + +func wrapOptionalError(context string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%s: %w", context, err) +} + +func validateStartedAgentUpgrade(paths agentUpgradePaths, signal *agentUpgradeSignal) error { + if signal.CandidatePath != paths.BluePath && signal.CandidatePath != paths.GreenPath { + return fmt.Errorf("AgentUpgrade was interrupted before selecting a candidate slot") + } + current, err := resolvedExecutable(paths.CurrentPath) + if err != nil { + return fmt.Errorf("resolve current upgraded agent binary: %w", err) + } + if current != signal.CandidatePath { + return fmt.Errorf("AgentUpgrade was interrupted before switching the candidate binary") + } + if !validNspawnMachine(signal.ActiveMachine) { + return fmt.Errorf("AgentUpgrade has no valid active nspawn machine") + } + nspawnPath := filepath.Join("/var/lib/machines", signal.ActiveMachine, "usr", "local", "bin", "aks-flex-node") + equal, err := filesHaveEqualSHA256(current, nspawnPath) + if err != nil { + return fmt.Errorf("verify synchronized nspawn agent binary: %w", err) + } + if !equal { + return fmt.Errorf("upgraded host and nspawn agent binaries do not match") + } + return nil +} + +func rollbackAgentUpgradeFiles(paths agentUpgradePaths, signal *agentUpgradeSignal) error { + lastGood, err := resolvedExecutable(paths.LastGoodPath) + if err != nil { + return fmt.Errorf("resolve last-good agent binary: %w", err) + } + if err := replaceSymlink(paths.CurrentPath, lastGood); err != nil { + return fmt.Errorf("restore last-good agent binary: %w", err) + } + if validNspawnMachine(signal.ActiveMachine) { + if err := synchronizeNspawnAgentBinary(lastGood, signal.ActiveMachine); err != nil { + return err + } + } + return nil +} + +func filesHaveEqualSHA256(firstPath, secondPath string) (bool, error) { + first, err := fileSHA256(firstPath) + if err != nil { + return false, err + } + second, err := fileSHA256(secondPath) + if err != nil { + return false, err + } + return first == second, nil +} + +func fileSHA256(path string) ([sha256.Size]byte, error) { + var digest [sha256.Size]byte + file, err := os.Open(path) //nolint:gosec // fixed root-owned agent paths + if err != nil { + return digest, err + } + defer file.Close() //nolint:errcheck // read result is authoritative + hasher := sha256.New() + limited := io.LimitReader(file, agentUpgradeMaxBinaryBytes+1) + n, err := io.Copy(hasher, limited) + if err != nil { + return digest, err + } + if n > agentUpgradeMaxBinaryBytes { + return digest, fmt.Errorf("agent binary exceeds %d-byte limit", agentUpgradeMaxBinaryBytes) + } + copy(digest[:], hasher.Sum(nil)) + return digest, nil +} diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go new file mode 100644 index 00000000..687460df --- /dev/null +++ b/pkg/daemon/agent_upgrade_binary.go @@ -0,0 +1,456 @@ +package daemon + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/Azure/AKSFlexNode/pkg/utils/utilio" +) + +const ( + agentUpgradeBinaryMode = 0o755 + agentUpgradeMaxArchiveBytes = 256 << 20 + agentUpgradeMaxBinaryBytes = 256 << 20 + agentUpgradeVerifyTimeout = 30 * time.Second +) + +type agentUpgradePaths struct { + BinaryPath string + BluePath string + GreenPath string + CurrentPath string + LastGoodPath string + SignalPath string +} + +// ensureAgentUpgradeLayout migrates a legacy direct binary into the blue slot. +// It is intentionally idempotent because bootstrap and daemon startup may both +// call it while converging an older installation. +func ensureAgentUpgradeLayout(ctx context.Context, log *slog.Logger, paths agentUpgradePaths) error { + if err := ctx.Err(); err != nil { + return err + } + if log == nil { + return fmt.Errorf("logger is nil") + } + if err := validateAgentUpgradePaths(paths); err != nil { + return err + } + productionPaths := paths == defaultAgentUpgradePaths() + if productionPaths && os.Geteuid() != 0 { + return fmt.Errorf("agent binary layout must be initialized as root") + } + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + return fmt.Errorf("create agent binary slot directory: %w", err) + } + + currentTarget, err := resolvedExecutable(paths.CurrentPath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resolve current agent binary: %w", err) + } + if errors.Is(err, os.ErrNotExist) { + seed, seedErr := initialAgentBinary(paths) + if seedErr != nil { + return fmt.Errorf("find initial agent binary: %w", seedErr) + } + if seed == paths.BinaryPath { + if err := copyExecutable(paths.BinaryPath, paths.BluePath); err != nil { + return fmt.Errorf("migrate legacy agent binary: %w", err) + } + seed = paths.BluePath + } + if err := replaceSymlink(paths.CurrentPath, seed); err != nil { + return fmt.Errorf("initialize current agent symlink: %w", err) + } + currentTarget = seed + } + + if _, err := resolvedExecutable(paths.LastGoodPath); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resolve last-good agent binary: %w", err) + } + if err := replaceSymlink(paths.LastGoodPath, currentTarget); err != nil { + return fmt.Errorf("initialize last-good agent symlink: %w", err) + } + } + + binaryTarget, err := filepath.EvalSymlinks(paths.BinaryPath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resolve compatibility agent binary: %w", err) + } + if errors.Is(err, os.ErrNotExist) || binaryTarget != currentTarget { + if err := replaceSymlink(paths.BinaryPath, paths.CurrentPath); err != nil { + return fmt.Errorf("initialize compatibility agent symlink: %w", err) + } + } + + if productionPaths { + if err := validateRootOwnedAgentUpgradePaths(paths); err != nil { + return err + } + } + log.Info("agent binary blue-green layout initialized", "current", paths.CurrentPath, "last_good", paths.LastGoodPath) + return nil +} + +func validateRootOwnedAgentUpgradePaths(paths agentUpgradePaths) error { + for _, path := range []string{ + filepath.Dir(paths.BluePath), + paths.BinaryPath, + paths.BluePath, + paths.GreenPath, + paths.CurrentPath, + paths.LastGoodPath, + paths.SignalPath, + } { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return fmt.Errorf("inspect ownership of %s: %w", path, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != 0 { + return fmt.Errorf("agent upgrade path %s is not root-owned", path) + } + } + return nil +} + +func validateAgentUpgradePaths(paths agentUpgradePaths) error { + values := []string{paths.BinaryPath, paths.BluePath, paths.GreenPath, paths.CurrentPath, paths.LastGoodPath, paths.SignalPath} + seen := make(map[string]struct{}, len(values)) + for _, value := range values { + if value == "" || !filepath.IsAbs(value) || filepath.Clean(value) != value { + return fmt.Errorf("invalid agent upgrade path %q", value) + } + if _, ok := seen[value]; ok { + return fmt.Errorf("duplicate agent upgrade path %q", value) + } + seen[value] = struct{}{} + } + return nil +} + +func initialAgentBinary(paths agentUpgradePaths) (string, error) { + for _, path := range []string{paths.BluePath, paths.GreenPath, paths.BinaryPath} { + if _, err := resolvedExecutable(path); err == nil { + return path, nil + } + } + return "", os.ErrNotExist +} + +func resolvedExecutable(path string) (string, error) { + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + info, err := os.Stat(resolved) + if err != nil { + return "", err + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return "", fmt.Errorf("%s is not a regular executable file", path) + } + return resolved, nil +} + +func copyExecutable(sourcePath, targetPath string) (err error) { + source, err := os.Open(sourcePath) //nolint:gosec // paths are fixed daemon configuration + if err != nil { + return err + } + defer func() { + if closeErr := source.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() + return utilio.InstallFileWithLimitedSize(targetPath, source, agentUpgradeBinaryMode, agentUpgradeMaxBinaryBytes) +} + +func replaceSymlink(linkPath, targetPath string) error { + if err := os.MkdirAll(filepath.Dir(linkPath), 0o750); err != nil { + return err + } + temp, err := os.CreateTemp(filepath.Dir(linkPath), ".aks-flex-node-link-*") + if err != nil { + return err + } + tempPath := temp.Name() + if err := temp.Close(); err != nil { + _ = os.Remove(tempPath) + return err + } + if err := os.Remove(tempPath); err != nil { + return err + } + defer os.Remove(tempPath) //nolint:errcheck // best-effort cleanup before/after rename + if err := os.Symlink(targetPath, tempPath); err != nil { + return err + } + return os.Rename(tempPath, linkPath) +} + +func parseAgentUpgradeSHA256(value string) ([sha256.Size]byte, error) { + var expected [sha256.Size]byte + value = strings.TrimSpace(value) + value = strings.TrimPrefix(value, "sha256:") + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != sha256.Size { + return expected, fmt.Errorf("expected SHA-256 must be exactly 64 hexadecimal characters") + } + copy(expected[:], decoded) + return expected, nil +} + +func validateAgentUpgradeURL(rawURL string) (*url.URL, error) { + parsed, err := url.ParseRequestURI(strings.TrimSpace(rawURL)) + if err != nil { + return nil, fmt.Errorf("invalid download URL") + } + if parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { + return nil, fmt.Errorf("download URL must be an HTTPS URL without user information") + } + return parsed, nil +} + +func redactedAgentUpgradeURL(parsed *url.URL) string { + redacted := *parsed + redacted.RawQuery = "" + redacted.Fragment = "" + return redacted.String() +} + +func expectedAgentArchiveMember() (string, error) { + switch runtime.GOARCH { + case "amd64", "arm64": + return "aks-flex-node-linux-" + runtime.GOARCH, nil + default: + return "", fmt.Errorf("unsupported agent upgrade architecture %q", runtime.GOARCH) + } +} + +// installAndSwitchAgentBinary downloads and verifies an archive before making +// either daemon symlink visible. The archive digest covers the compressed bytes. +func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, expectedDigest string, paths agentUpgradePaths) error { + return installAndSwitchAgentBinaryWithClient(ctx, log, newAgentUpgradeHTTPClient(), rawURL, expectedDigest, paths) +} + +func installAndSwitchAgentBinaryWithClient(ctx context.Context, log *slog.Logger, client *http.Client, rawURL, expectedDigest string, paths agentUpgradePaths) error { + parsedURL, err := validateAgentUpgradeURL(rawURL) + if err != nil { + return err + } + expected, err := parseAgentUpgradeSHA256(expectedDigest) + if err != nil { + return err + } + member, err := expectedAgentArchiveMember() + if err != nil { + return err + } + currentTarget, err := resolvedExecutable(paths.CurrentPath) + if err != nil { + return fmt.Errorf("resolve current agent binary: %w", err) + } + targetPath := paths.BluePath + if currentTarget == paths.BluePath { + targetPath = paths.GreenPath + } + + archivePath, err := downloadAgentUpgradeArchive(ctx, client, parsedURL, expected) + if err != nil { + return err + } + defer os.Remove(archivePath) //nolint:errcheck // temporary archive cleanup + // The inactive slot may still be the target of last-good. Point last-good + // at the verified running binary before replacing that slot. + if err := replaceSymlink(paths.LastGoodPath, currentTarget); err != nil { + return fmt.Errorf("protect current agent as last-good: %w", err) + } + if err := extractAgentUpgradeBinary(archivePath, member, targetPath); err != nil { + return err + } + if err := verifyAgentBinary(ctx, targetPath); err != nil { + return err + } + if err := replaceSymlink(paths.CurrentPath, targetPath); err != nil { + return fmt.Errorf("update current agent symlink: %w", err) + } + log.Info("staged upgraded agent binary", "url", redactedAgentUpgradeURL(parsedURL), "previous", currentTarget, "current", targetPath) + return nil +} + +func newAgentUpgradeHTTPClient() *http.Client { + return &http.Client{ + Timeout: 10 * time.Minute, + CheckRedirect: func(req *http.Request, _ []*http.Request) error { + if req.URL.Scheme != "https" { + return fmt.Errorf("redirect to non-HTTPS URL is not allowed") + } + return nil + }, + } +} + +func downloadAgentUpgradeArchive(ctx context.Context, client *http.Client, parsedURL *url.URL, expected [sha256.Size]byte) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedURL.String(), http.NoBody) + if err != nil { + return "", fmt.Errorf("create agent archive request: %w", err) + } + resp, err := client.Do(req) + if err != nil { + if ctx.Err() != nil { + return "", fmt.Errorf("download agent archive from %s: %w", redactedAgentUpgradeURL(parsedURL), ctx.Err()) + } + // Redirect targets and transport errors may contain credential-bearing + // URLs, so do not propagate the transport's error text. + return "", fmt.Errorf("download agent archive from %s failed", redactedAgentUpgradeURL(parsedURL)) + } + defer resp.Body.Close() //nolint:errcheck // response body cleanup + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("download agent archive from %s: HTTP status %d", redactedAgentUpgradeURL(parsedURL), resp.StatusCode) + } + if resp.ContentLength > agentUpgradeMaxArchiveBytes { + return "", fmt.Errorf("agent archive exceeds %d-byte limit", agentUpgradeMaxArchiveBytes) + } + + temp, err := os.CreateTemp("", "aks-flex-node-upgrade-*.tar.gz") + if err != nil { + return "", fmt.Errorf("create temporary agent archive: %w", err) + } + path := temp.Name() + ok := false + defer func() { + _ = temp.Close() + if !ok { + _ = os.Remove(path) + } + }() + hasher := sha256.New() + limited := io.LimitReader(resp.Body, agentUpgradeMaxArchiveBytes+1) + n, err := io.Copy(io.MultiWriter(temp, hasher), limited) + if err != nil { + return "", fmt.Errorf("read agent archive: %w", err) + } + if n > agentUpgradeMaxArchiveBytes { + return "", fmt.Errorf("agent archive exceeds %d-byte limit", agentUpgradeMaxArchiveBytes) + } + if !equalDigest(hasher.Sum(nil), expected[:]) { + return "", fmt.Errorf("agent archive SHA-256 does not match expected digest") + } + if err := temp.Close(); err != nil { + return "", fmt.Errorf("close temporary agent archive: %w", err) + } + ok = true + return path, nil +} + +func equalDigest(actual, expected []byte) bool { + if len(actual) != len(expected) { + return false + } + var different byte + for i := range actual { + different |= actual[i] ^ expected[i] + } + return different == 0 +} + +func extractAgentUpgradeBinary(archivePath, expectedMember, targetPath string) (err error) { + archive, err := os.Open(archivePath) //nolint:gosec // path is an internally created temporary file + if err != nil { + return fmt.Errorf("open agent archive: %w", err) + } + defer func() { + if closeErr := archive.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() + gz, err := gzip.NewReader(archive) + if err != nil { + return fmt.Errorf("decompress agent archive: %w", err) + } + defer gz.Close() //nolint:errcheck // read errors are reported while extracting + + found := false + // Bound total decompressed input as well as the selected member so gzip + // bombs hidden in unrelated archive members cannot consume unbounded work. + decompressed := &countingReader{reader: io.LimitReader(gz, 2*agentUpgradeMaxBinaryBytes+1)} + tarReader := tar.NewReader(decompressed) + for { + header, nextErr := tarReader.Next() + if errors.Is(nextErr, io.EOF) { + break + } + if nextErr != nil { + return fmt.Errorf("read agent archive: %w", nextErr) + } + if header.Name == "" || filepath.IsAbs(header.Name) || filepath.Clean(header.Name) != header.Name || strings.Contains(header.Name, `\`) || strings.HasPrefix(header.Name, ".."+string(filepath.Separator)) { + return fmt.Errorf("agent archive contains unsafe member name %q", header.Name) + } + if header.Name != expectedMember { + return fmt.Errorf("agent archive contains unexpected member %q", header.Name) + } + if found { + return fmt.Errorf("agent archive contains duplicate member %q", expectedMember) + } + if header.Typeflag != tar.TypeReg || header.Size <= 0 || header.Size > agentUpgradeMaxBinaryBytes { + return fmt.Errorf("agent archive member %q is not a valid bounded regular file", expectedMember) + } + if err := utilio.InstallFileWithLimitedSize(targetPath, tarReader, agentUpgradeBinaryMode, agentUpgradeMaxBinaryBytes); err != nil { + return fmt.Errorf("install upgraded agent binary: %w", err) + } + found = true + } + if decompressed.count > 2*agentUpgradeMaxBinaryBytes { + return fmt.Errorf("decompressed agent archive exceeds %d-byte limit", 2*agentUpgradeMaxBinaryBytes) + } + if !found { + return fmt.Errorf("agent archive does not contain expected member %q", expectedMember) + } + return nil +} + +type countingReader struct { + reader io.Reader + count int64 +} + +func (r *countingReader) Read(data []byte) (int, error) { + n, err := r.reader.Read(data) + r.count += int64(n) + return n, err +} + +func verifyAgentBinary(ctx context.Context, path string) error { + verifyCtx, cancel := context.WithTimeout(ctx, agentUpgradeVerifyTimeout) + defer cancel() + cmd := exec.CommandContext(verifyCtx, path, "version") //nolint:gosec // fixed verified binary path and argument + // Candidate output is untrusted and could expose host data in operation + // status. Only the command's success is relevant to verification. + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return fmt.Errorf("verify upgraded agent binary: %w", err) + } + return nil +} diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go new file mode 100644 index 00000000..c49e1861 --- /dev/null +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -0,0 +1,334 @@ +package daemon + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestEnsureAgentUpgradeLayoutMigratesLegacyBinaryIdempotently(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BinaryPath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.BinaryPath, []byte("legacy"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + for range 2 { + if err := ensureAgentUpgradeLayout(t.Context(), slog.Default(), paths); err != nil { + t.Fatalf("ensureAgentUpgradeLayout: %v", err) + } + } + + assertResolvedPath(t, paths.CurrentPath, paths.BluePath) + assertResolvedPath(t, paths.LastGoodPath, paths.BluePath) + assertResolvedPath(t, paths.BinaryPath, paths.BluePath) + data, err := os.ReadFile(paths.BluePath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != "legacy" { + t.Fatalf("blue slot = %q, want legacy", data) + } +} + +func TestParseAgentUpgradeSHA256(t *testing.T) { + t.Parallel() + + digest := strings.Repeat("ab", sha256.Size) + for _, value := range []string{digest, "sha256:" + digest} { + if _, err := parseAgentUpgradeSHA256(value); err != nil { + t.Fatalf("parseAgentUpgradeSHA256(%q): %v", value, err) + } + } + for _, value := range []string{"", "abc", strings.Repeat("z", 64)} { + if _, err := parseAgentUpgradeSHA256(value); err == nil { + t.Fatalf("parseAgentUpgradeSHA256(%q) error = nil", value) + } + } +} + +func TestValidateAgentUpgradeURL(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + value string + wantErr bool + }{ + "HTTPS": {value: "https://example.com/agent.tar.gz?sig=secret"}, + "HTTP": {value: "http://example.com/agent.tar.gz", wantErr: true}, + "file": {value: "file:///tmp/agent.tar.gz", wantErr: true}, + "userinfo": {value: "https://user:secret@example.com/agent.tar.gz", wantErr: true}, + "missing host": {value: "https:///agent.tar.gz", wantErr: true}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + _, err := validateAgentUpgradeURL(tt.value) + if (err != nil) != tt.wantErr { + t.Fatalf("validateAgentUpgradeURL() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestRedactedAgentUpgradeURLRemovesQuery(t *testing.T) { + t.Parallel() + + parsed, err := url.Parse("https://example.com/agent.tar.gz?sig=secret") + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := redactedAgentUpgradeURL(parsed); got != "https://example.com/agent.tar.gz" { + t.Fatalf("redacted URL = %q", got) + } +} + +func TestDownloadAgentUpgradeArchiveVerifiesDigestAndRedactsErrors(t *testing.T) { + t.Parallel() + + payload := []byte("archive") + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(payload) + })) + t.Cleanup(server.Close) + parsed, err := url.Parse(server.URL + "/agent.tar.gz?sig=secret") + if err != nil { + t.Fatalf("Parse: %v", err) + } + expected := sha256.Sum256(payload) + path, err := downloadAgentUpgradeArchive(t.Context(), server.Client(), parsed, expected) + if err != nil { + t.Fatalf("downloadAgentUpgradeArchive: %v", err) + } + t.Cleanup(func() { _ = os.Remove(path) }) + + wrong := sha256.Sum256([]byte("wrong")) + _, err = downloadAgentUpgradeArchive(t.Context(), server.Client(), parsed, wrong) + if err == nil { + t.Fatal("downloadAgentUpgradeArchive error = nil") + } + if strings.Contains(err.Error(), "secret") { + t.Fatalf("error leaked URL query: %v", err) + } +} + +func TestExtractAgentUpgradeBinary(t *testing.T) { + t.Parallel() + + member, err := expectedAgentArchiveMember() + if err != nil { + t.Skipf("unsupported test architecture: %v", err) + } + archivePath := filepath.Join(t.TempDir(), "agent.tar.gz") + binary := []byte("#!/bin/sh\nexit 0\n") + writeTestAgentArchive(t, archivePath, []testTarMember{{name: member, mode: 0o755, body: binary}}) + targetPath := filepath.Join(t.TempDir(), "agent") + if err := extractAgentUpgradeBinary(archivePath, member, targetPath); err != nil { + t.Fatalf("extractAgentUpgradeBinary: %v", err) + } + got, err := os.ReadFile(targetPath) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if !bytes.Equal(got, binary) { + t.Fatalf("binary = %q, want %q", got, binary) + } + info, err := os.Stat(targetPath) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if info.Mode().Perm() != agentUpgradeBinaryMode { + t.Fatalf("mode = %o, want %o", info.Mode().Perm(), agentUpgradeBinaryMode) + } +} + +func TestExtractAgentUpgradeBinaryRejectsUnsafeAndUnexpectedArchives(t *testing.T) { + t.Parallel() + + member := "aks-flex-node-linux-" + runtime.GOARCH + tests := map[string][]testTarMember{ + "missing member": {{name: "other", body: []byte("binary")}}, + "traversal": {{name: "../" + member, body: []byte("binary")}}, + "duplicate": { + {name: member, body: []byte("one")}, + {name: member, body: []byte("two")}, + }, + } + for name, members := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + archivePath := filepath.Join(t.TempDir(), "agent.tar.gz") + writeTestAgentArchive(t, archivePath, members) + err := extractAgentUpgradeBinary(archivePath, member, filepath.Join(t.TempDir(), "agent")) + if err == nil { + t.Fatal("extractAgentUpgradeBinary error = nil") + } + }) + } +} + +func TestInstallAndSwitchAgentBinarySwitchesAndProtectsLastGood(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + oldBinary := []byte("#!/bin/sh\nexit 0\n") + if err := os.WriteFile(paths.BluePath, oldBinary, 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.LastGoodPath); err != nil { + t.Fatalf("Symlink last-good: %v", err) + } + member, err := expectedAgentArchiveMember() + if err != nil { + t.Skipf("unsupported test architecture: %v", err) + } + goodArchive := filepath.Join(t.TempDir(), "good.tar.gz") + goodBinary := []byte("#!/bin/sh\nexit 0\n") + writeTestAgentArchive(t, goodArchive, []testTarMember{{name: member, body: goodBinary}}) + goodPayload, err := os.ReadFile(goodArchive) + if err != nil { + t.Fatalf("ReadFile good archive: %v", err) + } + badArchive := filepath.Join(t.TempDir(), "bad.tar.gz") + badBinary := []byte("#!/bin/sh\nexit 42\n") + writeTestAgentArchive(t, badArchive, []testTarMember{{name: member, body: badBinary}}) + badPayload, err := os.ReadFile(badArchive) + if err != nil { + t.Fatalf("ReadFile bad archive: %v", err) + } + + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/bad.tar.gz" { + _, _ = w.Write(badPayload) + return + } + _, _ = w.Write(goodPayload) + })) + t.Cleanup(server.Close) + goodDigest := sha256.Sum256(goodPayload) + if err := installAndSwitchAgentBinaryWithClient(t.Context(), slog.Default(), server.Client(), server.URL+"/good.tar.gz", fmt.Sprintf("%x", goodDigest), paths); err != nil { + t.Fatalf("install good candidate: %v", err) + } + assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) + assertResolvedPath(t, paths.LastGoodPath, paths.BluePath) + + badDigest := sha256.Sum256(badPayload) + if err := installAndSwitchAgentBinaryWithClient(t.Context(), slog.Default(), server.Client(), server.URL+"/bad.tar.gz", fmt.Sprintf("%x", badDigest), paths); err == nil { + t.Fatal("install bad candidate error = nil") + } + assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) + // The failed candidate overwrote the inactive blue slot, so last-good must + // have moved to the still-running verified green slot first. + assertResolvedPath(t, paths.LastGoodPath, paths.GreenPath) +} + +func TestInstallAndSwitchAgentBinaryRejectsInvalidInputsWithoutSwitching(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.BluePath, []byte("old"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink: %v", err) + } + + err := installAndSwitchAgentBinary(context.Background(), slog.Default(), "http://example.com/agent.tar.gz", strings.Repeat("0", 64), paths) + if err == nil { + t.Fatal("installAndSwitchAgentBinary error = nil") + } + assertResolvedPath(t, paths.CurrentPath, paths.BluePath) +} + +type testTarMember struct { + name string + mode int64 + body []byte +} + +func writeTestAgentArchive(t *testing.T, path string, members []testTarMember) { + t.Helper() + file, err := os.Create(path) //nolint:gosec // test-owned temporary path + if err != nil { + t.Fatalf("Create: %v", err) + } + gz := gzip.NewWriter(file) + tarWriter := tar.NewWriter(gz) + for _, member := range members { + mode := member.mode + if mode == 0 { + mode = 0o755 + } + if err := tarWriter.WriteHeader(&tar.Header{Name: member.name, Mode: mode, Size: int64(len(member.body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + if _, err := io.Copy(tarWriter, bytes.NewReader(member.body)); err != nil { + t.Fatalf("Write: %v", err) + } + } + if err := tarWriter.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close file: %v", err) + } +} + +func testAgentUpgradePaths(t *testing.T) agentUpgradePaths { + t.Helper() + dir := t.TempDir() + return agentUpgradePaths{ + BinaryPath: filepath.Join(dir, "bin", "aks-flex-node"), + BluePath: filepath.Join(dir, "lib", "aks-flex-node-blue"), + GreenPath: filepath.Join(dir, "lib", "aks-flex-node-green"), + CurrentPath: filepath.Join(dir, "lib", "aks-flex-node-current"), + LastGoodPath: filepath.Join(dir, "lib", "aks-flex-node-last-good"), + SignalPath: filepath.Join(dir, "etc", "agent-upgrade-signal.json"), + } +} + +func assertResolvedPath(t *testing.T, path, want string) { + t.Helper() + got, err := filepath.EvalSymlinks(path) + if err != nil { + t.Fatalf("EvalSymlinks(%s): %v", path, err) + } + if got != want { + t.Fatalf("resolved %s = %s, want %s", path, got, want) + } +} + +func Example_redactedAgentUpgradeURL() { + parsed, _ := url.Parse("https://example.com/agent.tar.gz?sig=secret") + fmt.Println(redactedAgentUpgradeURL(parsed)) + // Output: https://example.com/agent.tar.gz +} diff --git a/pkg/daemon/agent_upgrade_test.go b/pkg/daemon/agent_upgrade_test.go new file mode 100644 index 00000000..2e3f1997 --- /dev/null +++ b/pkg/daemon/agent_upgrade_test.go @@ -0,0 +1,390 @@ +package daemon + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + + machinav1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + agentdaemon "github.com/Azure/unbounded/pkg/agent/daemon" +) + +func TestNewDaemonInstanceID(t *testing.T) { + t.Parallel() + + first, err := newDaemonInstanceID() + if err != nil { + t.Fatalf("newDaemonInstanceID: %v", err) + } + second, err := newDaemonInstanceID() + if err != nil { + t.Fatalf("newDaemonInstanceID: %v", err) + } + if first == "" || second == "" || first == second { + t.Fatalf("instance IDs = %q, %q", first, second) + } +} + +func TestParseAgentUpgradeRequest(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + parameters map[string]string + wantErr string + }{ + "valid": { + parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz?sig=secret", + agentUpgradeSHA256Parameter: strings.Repeat("a", 64), + }, + }, + "missing URL": { + parameters: map[string]string{agentUpgradeSHA256Parameter: strings.Repeat("a", 64)}, + wantErr: agentUpgradeDownloadURLParameter, + }, + "HTTP URL": { + parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "http://example.com/agent.tar.gz", + agentUpgradeSHA256Parameter: strings.Repeat("a", 64), + }, + wantErr: "HTTPS", + }, + "missing digest": { + parameters: map[string]string{agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz"}, + wantErr: agentUpgradeSHA256Parameter, + }, + "invalid digest": { + parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + agentUpgradeSHA256Parameter: "bad", + }, + wantErr: "64 hexadecimal", + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + _, err := parseAgentUpgradeRequest(tt.parameters) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("parseAgentUpgradeRequest: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %v, want substring %q", err, tt.wantErr) + } + }) + } +} + +func TestHostAgentUpgradeExecutorRecordPendingIsIdempotent(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "agent-upgrade.json") + executor := &hostAgentUpgradeExecutor{ + state: &fakeNodeOperator{state: &State{ActiveMachine: "kube1"}}, + signals: agentUpgradeSignalStore{path: path}, + } + if err := executor.RecordPending(t.Context(), "operation-1"); err != nil { + t.Fatalf("first RecordPending: %v", err) + } + if err := executor.RecordPending(t.Context(), "operation-1"); !errors.Is(err, errAgentUpgradeAlreadyPending) { + t.Fatalf("second RecordPending error = %v, want errAgentUpgradeAlreadyPending", err) + } + if err := executor.RecordPending(t.Context(), "operation-2"); err == nil || !strings.Contains(err.Error(), "operation-1") { + t.Fatalf("competing RecordPending error = %v", err) + } +} + +func TestAgentUpgradeSignalStoreLifecycle(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "signals", "agent-upgrade.json") + store := agentUpgradeSignalStore{path: path} + if err := store.recordPending("operation-1", "kube1", "instance-1"); err != nil { + t.Fatalf("recordPending: %v", err) + } + if err := store.recordCandidate("/slots/green"); err != nil { + t.Fatalf("recordCandidate: %v", err) + } + if err := store.recordFailure("rolled back"); err != nil { + t.Fatalf("recordFailure: %v", err) + } + signal, err := store.read() + if err != nil { + t.Fatalf("read: %v", err) + } + if signal == nil || signal.OperationName != "operation-1" || signal.ActiveMachine != "kube1" || signal.CandidatePath != "/slots/green" || signal.InitiatingDaemonInstance != "instance-1" || signal.Failure != "rolled back" || !signal.RecoveryRequired { + t.Fatalf("signal = %#v", signal) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("signal mode = %o, want 600", info.Mode().Perm()) + } + if err := store.clear(); err != nil { + t.Fatalf("clear: %v", err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("signal still exists: %v", err) + } +} + +func TestAgentUpgradeSignalStoreRejectsInvalidMachine(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "signal.json") + if err := os.WriteFile(path, []byte(`{"operationName":"operation-1","activeMachine":"../../etc"}`), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + _, err := (agentUpgradeSignalStore{path: path}).read() + if err == nil || !strings.Contains(err.Error(), "invalid active machine") { + t.Fatalf("error = %v", err) + } +} + +func TestHostAgentUpgradeExecutorSchedulesRestartOutsideService(t *testing.T) { + t.Parallel() + + executor := &hostAgentUpgradeExecutor{ + runSystemdRun: func(_ context.Context, args ...string) error { + joined := strings.Join(args, " ") + for _, expected := range []string{ + "--collect", + "--on-active=1s", + "--unit=aks-flex-node-agent-upgrade-restart-", + "/usr/bin/systemctl restart " + ServiceUnitName, + } { + if !strings.Contains(joined, expected) { + t.Fatalf("systemd-run args %q do not contain %q", joined, expected) + } + } + return nil + }, + } + if err := executor.Restart(t.Context()); err != nil { + t.Fatalf("Restart: %v", err) + } +} + +func TestHostAgentUpgradeExecutorRestartRejectsPreCanceledContext(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + called := false + executor := &hostAgentUpgradeExecutor{runSystemdRun: func(context.Context, ...string) error { + called = true + return nil + }} + if err := executor.Restart(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("Restart error = %v, want context.Canceled", err) + } + if called { + t.Fatal("systemctl called with an already canceled context") + } +} + +func TestHostAgentUpgradeExecutorAbortUsesCleanupContext(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + for path, content := range map[string]string{paths.BluePath: "blue", paths.GreenPath: "green"} { + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } + } + if err := os.Symlink(paths.GreenPath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.LastGoodPath); err != nil { + t.Fatalf("Symlink last-good: %v", err) + } + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.recordPending("operation-1", "", "instance-1"); err != nil { + t.Fatalf("recordPending: %v", err) + } + executor := &hostAgentUpgradeExecutor{paths: paths, signals: signals, instanceID: "instance-1"} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := executor.Abort(ctx); err != nil { + t.Fatalf("Abort: %v", err) + } + assertResolvedPath(t, paths.CurrentPath, paths.BluePath) + if signal, err := signals.read(); err != nil || signal != nil { + t.Fatalf("signal after Abort = %#v, %v", signal, err) + } +} + +func TestHostAgentUpgradeExecutorAbortPreservesSignalOnRollbackFailure(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.recordPending("operation-1", "", "instance-1"); err != nil { + t.Fatalf("recordPending: %v", err) + } + executor := &hostAgentUpgradeExecutor{paths: paths, signals: signals, instanceID: "instance-1"} + if err := executor.Abort(t.Context()); err == nil { + t.Fatal("Abort error = nil") + } + if signal, err := signals.read(); err != nil || signal == nil { + t.Fatalf("signal after failed Abort = %#v, %v", signal, err) + } +} + +func TestPublishAgentUpgradeSignalPreservesSignalWhenRollbackFails(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.BluePath, []byte("candidate"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink: %v", err) + } + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.write(agentUpgradeSignal{ + OperationName: "operation-1", + ActiveMachine: "kube1", + CandidatePath: paths.BluePath, + InitiatingDaemonInstance: "other-instance", + }); err != nil { + t.Fatalf("write signal: %v", err) + } + executor := &hostAgentUpgradeExecutor{paths: paths, signals: signals, instanceID: "current-instance"} + if err := publishAndClearAgentUpgradeSignal(t.Context(), slog.Default(), nil, executor); err == nil { + t.Fatal("publishAndClearAgentUpgradeSignal error = nil") + } + if signal, err := signals.read(); err != nil || signal == nil { + t.Fatalf("signal after failed rollback = %#v, %v", signal, err) + } +} + +func TestPublishAgentUpgradeFailureRestartsIntoLastGoodBeforeClearingSignal(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + for path, content := range map[string]string{paths.BluePath: "candidate", paths.GreenPath: "last-good"} { + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } + } + if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + if err := os.Symlink(paths.GreenPath, paths.LastGoodPath); err != nil { + t.Fatalf("Symlink last-good: %v", err) + } + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.write(agentUpgradeSignal{ + OperationName: "operation-1", + CandidatePath: paths.BluePath, + InitiatingDaemonInstance: "initiator", + RecoveryRequired: true, + Failure: "candidate failed", + }); err != nil { + t.Fatalf("write signal: %v", err) + } + restarts := 0 + finished := 0 + executor := &hostAgentUpgradeExecutor{ + paths: paths, + signals: signals, + instanceID: "candidate-instance", + runSystemdRun: func(context.Context, ...string) error { + restarts++ + return nil + }, + finishMachineOperation: func(_ context.Context, _ client.Client, _ agentdaemon.MachineOperation, result agentdaemon.MachineOperationResult[int64]) error { + finished++ + if result.Phase != machinav1alpha3.OperationPhaseFailed { + t.Fatalf("phase = %s, want Failed", result.Phase) + } + return nil + }, + runningExecutable: func() (string, error) { return paths.BluePath, nil }, + } + if err := publishAndClearAgentUpgradeSignal(t.Context(), slog.Default(), nil, executor); err != nil { + t.Fatalf("publish candidate recovery: %v", err) + } + assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) + if signal, err := signals.read(); err != nil || signal == nil { + t.Fatalf("signal cleared before last-good startup: %#v, %v", signal, err) + } + if restarts != 1 || finished != 1 { + t.Fatalf("restarts = %d, finished = %d", restarts, finished) + } + + executor.instanceID = "last-good-instance" + executor.runningExecutable = func() (string, error) { return paths.GreenPath, nil } + if err := publishAndClearAgentUpgradeSignal(t.Context(), slog.Default(), nil, executor); err != nil { + t.Fatalf("publish last-good recovery: %v", err) + } + if signal, err := signals.read(); err != nil || signal != nil { + t.Fatalf("signal after last-good startup = %#v, %v", signal, err) + } + if restarts != 1 || finished != 2 { + t.Fatalf("restarts = %d, finished = %d", restarts, finished) + } +} + +func TestPublishAgentUpgradeSignalIgnoresInitiatingProcess(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.recordPending("operation-1", "kube1", "current-instance"); err != nil { + t.Fatalf("recordPending: %v", err) + } + executor := &hostAgentUpgradeExecutor{paths: paths, signals: signals, instanceID: "current-instance"} + if err := publishAndClearAgentUpgradeSignal(t.Context(), slog.Default(), nil, executor); err != nil { + t.Fatalf("publishAndClearAgentUpgradeSignal: %v", err) + } + if signal, err := signals.read(); err != nil || signal == nil { + t.Fatalf("initiating process consumed signal: %#v, %v", signal, err) + } +} + +func TestFilesHaveEqualSHA256(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + first := filepath.Join(dir, "first") + second := filepath.Join(dir, "second") + if err := os.WriteFile(first, []byte("same"), 0o600); err != nil { + t.Fatalf("WriteFile first: %v", err) + } + if err := os.WriteFile(second, []byte("same"), 0o600); err != nil { + t.Fatalf("WriteFile second: %v", err) + } + equal, err := filesHaveEqualSHA256(first, second) + if err != nil || !equal { + t.Fatalf("filesHaveEqualSHA256 = %v, %v", equal, err) + } + if err := os.WriteFile(second, []byte("different"), 0o600); err != nil { + t.Fatalf("WriteFile second: %v", err) + } + equal, err = filesHaveEqualSHA256(first, second) + if err != nil || equal { + t.Fatalf("filesHaveEqualSHA256 = %v, %v", equal, err) + } +} diff --git a/pkg/daemon/assets/aks-flex-node-agent-recovery.service b/pkg/daemon/assets/aks-flex-node-agent-recovery.service new file mode 100644 index 00000000..d579d5ac --- /dev/null +++ b/pkg/daemon/assets/aks-flex-node-agent-recovery.service @@ -0,0 +1,6 @@ +[Unit] +Description=Recover AKS Flex Node Agent to last-known-good binary + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/aks-flex-node/aks-flex-node-recovery.sh diff --git a/pkg/daemon/assets/aks-flex-node-agent.service b/pkg/daemon/assets/aks-flex-node-agent.service index 628a77ce..37e09a62 100644 --- a/pkg/daemon/assets/aks-flex-node-agent.service +++ b/pkg/daemon/assets/aks-flex-node-agent.service @@ -2,7 +2,8 @@ Description=AKS Flex Node Agent After=network-online.target Wants=network-online.target -# Restart on failure to enable auto-recovery +OnFailure=aks-flex-node-agent-recovery.service +# Trigger last-good recovery after the upgraded daemon repeatedly fails. StartLimitIntervalSec=300 StartLimitBurst=5 diff --git a/pkg/daemon/assets/aks-flex-node-recovery.sh b/pkg/daemon/assets/aks-flex-node-recovery.sh new file mode 100644 index 00000000..71d3fa86 --- /dev/null +++ b/pkg/daemon/assets/aks-flex-node-recovery.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +signal=/etc/aks-flex-node/agent-upgrade-signal.json +last_good="$(readlink -f /usr/local/lib/aks-flex-node/aks-flex-node-last-good || true)" + +# An ordinary daemon failure must not change the selected binary. +if [[ ! -f "${signal}" ]]; then + exit 0 +fi +if [[ -z "${last_good}" || ! -x "${last_good}" ]]; then + echo "no executable last-known-good AKS Flex Node agent binary" >&2 + exit 1 +fi + +"${last_good}" recover-agent-upgrade \ + --message "upgraded daemon failed repeatedly; restored last-good binary" +systemctl reset-failed aks-flex-node-agent.service +systemctl --no-block restart aks-flex-node-agent.service diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 5bd87e35..e1c891d1 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -69,6 +69,14 @@ func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { if err != nil { return err } + upgrades, err := newHostAgentUpgradeExecutor(log, operator) + if err != nil { + return err + } + directClient, err := client.New(restCfg, client.Options{Scheme: newScheme()}) + if err != nil { + return fmt.Errorf("create direct Kubernetes client: %w", err) + } repaves, err := newRepaveReconciler(repaveReconcilerOptions{ Log: log, Machines: machines, @@ -87,6 +95,7 @@ func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { AKSMachineName: aksMachineName, MachineOperationMode: cfg.Agent.MachineOperationMode, Operator: repaves.operator, + AgentUpgrade: upgrades, }) if err != nil { return err @@ -95,7 +104,22 @@ func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { return fmt.Errorf("setup daemon controller: %w", err) } + publishCtx, stopPublisher := context.WithCancel(ctx) + defer stopPublisher() + go func() { + // Success is only safe to publish after manager startup has reached cache + // readiness. If startup fails, systemd retains the signal and recovers. + if !mgr.GetCache().WaitForCacheSync(publishCtx) { + return + } + if err := publishAndClearAgentUpgradeSignal(publishCtx, log, directClient, upgrades); err != nil { + log.Warn("failed to publish AgentUpgrade startup result", "error", err) + } + retryAgentUpgradeSignal(publishCtx, log, directClient, upgrades) + }() + err = mgr.Start(ctx) + stopPublisher() repaves.log.Info("daemon shutting down") return err } diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index 822d01b1..d0b40698 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -14,13 +14,21 @@ import ( ) const ( - ServiceUnitName = "aks-flex-node-agent.service" - systemdSystemDir = "/etc/systemd/system" + ServiceUnitName = "aks-flex-node-agent.service" + recoveryServiceUnitName = "aks-flex-node-agent-recovery.service" + recoveryScriptPath = "/usr/local/lib/aks-flex-node/aks-flex-node-recovery.sh" + systemdSystemDir = "/etc/systemd/system" ) //go:embed assets/aks-flex-node-agent.service var serviceUnitContent []byte +//go:embed assets/aks-flex-node-agent-recovery.service +var recoveryServiceUnitContent []byte + +//go:embed assets/aks-flex-node-recovery.sh +var recoveryScriptContent []byte + type installServiceTask struct { log *slog.Logger } @@ -33,9 +41,22 @@ func InstallService(log *slog.Logger) phases.Task { func (t *installServiceTask) Name() string { return "install-service" } func (t *installServiceTask) Do(ctx context.Context) error { - unitPath := filepath.Join(systemdSystemDir, ServiceUnitName) - if err := utilio.WriteFile(unitPath, serviceUnitContent, 0o644); err != nil { //nolint:gosec // service files must be world-readable - return fmt.Errorf("write %s: %w", unitPath, err) + if err := ensureAgentUpgradeLayout(ctx, t.log, defaultAgentUpgradePaths()); err != nil { + return fmt.Errorf("initialize agent binary layout: %w", err) + } + assets := []struct { + path string + content []byte + mode os.FileMode + }{ + {path: filepath.Join(systemdSystemDir, ServiceUnitName), content: serviceUnitContent, mode: 0o644}, + {path: filepath.Join(systemdSystemDir, recoveryServiceUnitName), content: recoveryServiceUnitContent, mode: 0o644}, + {path: recoveryScriptPath, content: recoveryScriptContent, mode: 0o750}, + } + for _, asset := range assets { + if err := utilio.WriteFile(asset.path, asset.content, asset.mode); err != nil { + return fmt.Errorf("write %s: %w", asset.path, err) + } } if err := utilexec.ReloadSystemd(ctx, t.log); err != nil { @@ -71,9 +92,15 @@ func (t *uninstallServiceTask) Do(ctx context.Context) error { t.log.Warn("failed to disable service (may not be enabled)", "unit", ServiceUnitName, "error", err) } - unitPath := filepath.Join(systemdSystemDir, ServiceUnitName) - if err := os.Remove(unitPath); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("remove %s: %w", unitPath, err) + for _, path := range []string{ + filepath.Join(systemdSystemDir, ServiceUnitName), + filepath.Join(systemdSystemDir, recoveryServiceUnitName), + recoveryScriptPath, + defaultAgentUpgradePaths().SignalPath, + } { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s: %w", path, err) + } } if err := utilexec.ReloadSystemd(ctx, t.log); err != nil { diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go new file mode 100644 index 00000000..c7053393 --- /dev/null +++ b/pkg/daemon/lifecycle_test.go @@ -0,0 +1,30 @@ +package daemon + +import ( + "strings" + "testing" +) + +func TestAgentServiceIncludesUpgradeRecovery(t *testing.T) { + t.Parallel() + + service := string(serviceUnitContent) + if !strings.Contains(service, "OnFailure="+recoveryServiceUnitName) { + t.Fatalf("service does not activate %s on failure", recoveryServiceUnitName) + } + if !strings.Contains(string(recoveryServiceUnitContent), "ExecStart="+recoveryScriptPath) { + t.Fatalf("recovery service does not execute %s", recoveryScriptPath) + } + script := string(recoveryScriptContent) + for _, expected := range []string{ + "recover-agent-upgrade", + "aks-flex-node-agent.service", + "aks-flex-node-last-good", + "agent-upgrade-signal.json", + "systemctl --no-block restart", + } { + if !strings.Contains(script, expected) { + t.Fatalf("recovery script does not contain %q", expected) + } + } +} diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index 94f7fbf6..dc32a718 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -2,6 +2,7 @@ package daemon import ( "context" + "errors" "fmt" "log/slog" @@ -25,11 +26,13 @@ type machineOperationReconcilerOptions struct { AKSMachineName string MachineOperationMode string Operator nodeOperator + AgentUpgrade agentUpgradeExecutor } type machineOperationHandlers struct { - log *slog.Logger - operator nodeOperator + log *slog.Logger + operator nodeOperator + agentUpgrade agentUpgradeExecutor } // machineOperationReconciler runs MachineOperations when the Machina CRD is available. @@ -46,6 +49,9 @@ func machineOperationReconciler( if opts.Operator == nil { return nil, fmt.Errorf("node operator is nil") } + if opts.AgentUpgrade == nil { + return nil, fmt.Errorf("agent upgrade executor is nil") + } if opts.MachineOperationMode == "" { opts.MachineOperationMode = machineOperationModeAuto } @@ -73,14 +79,14 @@ func machineOperationReconciler( return nil, fmt.Errorf("AKS machine name is empty") } - handlers := &machineOperationHandlers{log: opts.Log, operator: opts.Operator} + handlers := &machineOperationHandlers{log: opts.Log, operator: opts.Operator, agentUpgrade: opts.AgentUpgrade} reconciler, err := daemon.NewMachinaMachineOperationReconciler( opts.Client, opts.NodeName, opts.AKSMachineName, daemon.MachineOperationHandlers{ machinav1alpha3.OperationNodeReboot: handlers.reconcileNodeReboot, - machinav1alpha3.OperationAgentUpgrade: handlers.unsupportedOperation, + machinav1alpha3.OperationAgentUpgrade: handlers.reconcileAgentUpgrade, machinav1alpha3.OperationAgentReset: handlers.reconcileAgentReset, }, ) @@ -136,6 +142,72 @@ func (h *machineOperationHandlers) reconcileNodeReboot( return ctrl.Result{}, nil } +func (h *machineOperationHandlers) reconcileAgentUpgrade( + ctx context.Context, + store daemon.MachineOperationStore[int64], + op daemon.MachineOperation, +) (ctrl.Result, error) { + request, err := parseAgentUpgradeRequest(op.Parameters) + if err != nil { + return h.finishFailedMachineOperation(ctx, store, op, "InvalidParameters", err.Error()) + } + if err := store.MarkInProgress(ctx, op, "staging upgraded AKS Flex Node agent binary"); err != nil { + return ctrl.Result{}, fmt.Errorf("mark AgentUpgrade MachineOperation in progress: %w", err) + } + if err := h.agentUpgrade.RecordPending(ctx, op.Name); err != nil { + if errors.Is(err, errAgentUpgradeAlreadyPending) { + // An InProgress status event can already be queued before the delayed + // daemon restart. The durable signal proves this operation was staged + // by an earlier reconciliation. + return ctrl.Result{}, nil + } + return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", err.Error()) + } + if err := h.agentUpgrade.Stage(ctx, request); err != nil { + if abortErr := h.agentUpgrade.Abort(ctx); abortErr != nil { + return h.beginAgentUpgradeRecovery(ctx, op, err, abortErr) + } + return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", err.Error()) + } + if err := h.agentUpgrade.Restart(ctx); err != nil { + if abortErr := h.agentUpgrade.Abort(ctx); abortErr != nil { + return h.beginAgentUpgradeRecovery(ctx, op, err, abortErr) + } + return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", "failed to restart upgraded agent daemon") + } + // The restarted daemon publishes success after proving the candidate can + // initialize its Kubernetes client and controller. + return ctrl.Result{}, nil +} + +func (h *machineOperationHandlers) beginAgentUpgradeRecovery( + ctx context.Context, + op daemon.MachineOperation, + executionErr, rollbackErr error, +) (ctrl.Result, error) { + message := fmt.Sprintf("AgentUpgrade execution failed and requires recovery: %v", executionErr) + recordErr := h.agentUpgrade.RecordFailure(message) + if recordErr != nil { + h.log.Error("failed to annotate durable AgentUpgrade recovery signal", "operation", op.Name, "error", recordErr) + } + h.log.Error("AgentUpgrade rollback failed; restarting daemon for durable recovery", + "operation", op.Name, + "error", rollbackErr, + ) + cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) + defer cancel() + restartErr := h.agentUpgrade.Restart(cleanupCtx) + if restartErr != nil { + // If the failure annotation succeeded, the initiating daemon's signal + // loop can retry recovery without waiting for another reconciliation. + h.log.Error("failed to restart daemon for AgentUpgrade recovery", "operation", op.Name, "error", restartErr) + } + if recordErr != nil && restartErr != nil { + return ctrl.Result{}, errors.Join(recordErr, restartErr) + } + return ctrl.Result{}, nil +} + func (h *machineOperationHandlers) reconcileAgentReset( ctx context.Context, store daemon.MachineOperationStore[int64], diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index dffa321a..d2a4f6da 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -67,6 +67,7 @@ func TestMachineOperationReconcilerDisableModeSkipsDiscovery(t *testing.T) { Log: slog.Default(), MachineOperationMode: machineOperationModeDisable, Operator: &fakeNodeOperator{}, + AgentUpgrade: &fakeAgentUpgradeExecutor{}, }) if err != nil { t.Fatalf("machineOperationReconciler: %v", err) @@ -99,11 +100,20 @@ func TestMachineOperationReconcilerRequiresDependencies(t *testing.T) { }, "missing operator": { opts: machineOperationReconcilerOptions{ - Client: fake.NewClientBuilder().Build(), - Log: slog.Default(), + Client: fake.NewClientBuilder().Build(), + Log: slog.Default(), + AgentUpgrade: &fakeAgentUpgradeExecutor{}, }, wantErr: "node operator is nil", }, + "missing agent upgrade executor": { + opts: machineOperationReconcilerOptions{ + Client: fake.NewClientBuilder().Build(), + Log: slog.Default(), + Operator: &fakeNodeOperator{}, + }, + wantErr: "agent upgrade executor is nil", + }, } for name, tt := range tests { @@ -151,6 +161,7 @@ func TestMachineOperationReconcilerEnabledRequiresNames(t *testing.T) { NodeName: tt.nodeName, AKSMachineName: tt.aksMachineName, Operator: &fakeNodeOperator{}, + AgentUpgrade: &fakeAgentUpgradeExecutor{}, }) if err == nil { t.Fatal("machineOperationReconciler error = nil, want error") @@ -278,6 +289,120 @@ func TestMachineOperationHandlersUnsupportedOperation(t *testing.T) { } } +func TestMachineOperationHandlersAgentUpgrade(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{} + store := &fakeMachineOperationStore{} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + digest := strings.Repeat("a", 64) + op := daemon.MachineOperation{ + Name: "upgrade-1", + Kind: machinav1alpha3.OperationAgentUpgrade, + Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz?sig=secret", + agentUpgradeSHA256Parameter: digest, + }, + } + + if _, err := target.reconcileAgentUpgrade(t.Context(), store, op); err != nil { + t.Fatalf("reconcileAgentUpgrade: %v", err) + } + if !store.inProgress || !upgrader.pending || !upgrader.staged || !upgrader.restarted { + t.Fatalf("upgrade calls = inProgress:%v pending:%v staged:%v restarted:%v", store.inProgress, upgrader.pending, upgrader.staged, upgrader.restarted) + } + if store.result.Phase != "" { + t.Fatalf("phase = %s, want non-terminal until daemon restart", store.result.Phase) + } +} + +func TestMachineOperationHandlersAgentUpgradeInvalidParameters(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{} + store := &fakeMachineOperationStore{} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + + if _, err := target.reconcileAgentUpgrade(t.Context(), store, daemon.MachineOperation{Name: "upgrade-1", Kind: machinav1alpha3.OperationAgentUpgrade}); err != nil { + t.Fatalf("reconcileAgentUpgrade: %v", err) + } + if store.result.Phase != machinav1alpha3.OperationPhaseFailed || store.result.Reason != "InvalidParameters" { + t.Fatalf("result = %#v, want InvalidParameters failure", store.result) + } + if upgrader.pending || upgrader.staged || upgrader.restarted { + t.Fatal("executor called for invalid parameters") + } +} + +func TestMachineOperationHandlersAgentUpgradeDuplicateReconcileIsNoop(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{pendingErr: errAgentUpgradeAlreadyPending} + store := &fakeMachineOperationStore{} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + op := daemon.MachineOperation{Name: "upgrade-1", Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + agentUpgradeSHA256Parameter: strings.Repeat("a", 64), + }} + + if _, err := target.reconcileAgentUpgrade(t.Context(), store, op); err != nil { + t.Fatalf("reconcileAgentUpgrade: %v", err) + } + if upgrader.staged || upgrader.restarted || upgrader.aborted { + t.Fatal("duplicate reconciliation executed upgrade actions") + } + if store.result.Phase != "" { + t.Fatalf("duplicate reconciliation produced terminal phase %s", store.result.Phase) + } +} + +func TestMachineOperationHandlersAgentUpgradeStageFailureRollsBack(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{stageErr: errors.New("bad archive")} + store := &fakeMachineOperationStore{} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + op := daemon.MachineOperation{Name: "upgrade-1", Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + agentUpgradeSHA256Parameter: strings.Repeat("a", 64), + }} + + if _, err := target.reconcileAgentUpgrade(t.Context(), store, op); err != nil { + t.Fatalf("reconcileAgentUpgrade: %v", err) + } + if !upgrader.aborted { + t.Fatal("Abort was not called") + } + if store.result.Phase != machinav1alpha3.OperationPhaseFailed || store.result.Message != "bad archive" { + t.Fatalf("result = %#v", store.result) + } +} + +func TestMachineOperationHandlersAgentUpgradeAbortFailureStartsRecovery(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{ + stageErr: errors.New("stage failed"), + abortErr: errors.New("rollback failed"), + } + store := &fakeMachineOperationStore{} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + op := daemon.MachineOperation{Name: "upgrade-1", Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + agentUpgradeSHA256Parameter: strings.Repeat("a", 64), + }} + + if _, err := target.reconcileAgentUpgrade(t.Context(), store, op); err != nil { + t.Fatalf("reconcileAgentUpgrade: %v", err) + } + if upgrader.failure == "" || !upgrader.restarted { + t.Fatalf("recovery failure = %q, restarted = %v", upgrader.failure, upgrader.restarted) + } + if store.result.Phase != "" { + t.Fatalf("phase = %s, want recovery to publish terminal status", store.result.Phase) + } +} + func TestMachineOperationHandlersAgentReset(t *testing.T) { t.Parallel() @@ -342,6 +467,45 @@ func TestMachineOperationHandlersAgentResetStopFailure(t *testing.T) { } } +type fakeAgentUpgradeExecutor struct { + pending bool + staged bool + aborted bool + restarted bool + failure string + pendingErr error + stageErr error + abortErr error + restartErr error +} + +func (f *fakeAgentUpgradeExecutor) RecordPending(context.Context, string) error { + f.pending = true + return f.pendingErr +} + +func (f *fakeAgentUpgradeExecutor) RecordFailure(message string) error { + f.failure = message + return nil +} + +func (f *fakeAgentUpgradeExecutor) Stage(context.Context, agentUpgradeRequest) error { + f.staged = true + return f.stageErr +} + +func (f *fakeAgentUpgradeExecutor) Abort(context.Context) error { + f.aborted = true + return f.abortErr +} + +func (f *fakeAgentUpgradeExecutor) Restart(context.Context) error { + f.restarted = true + return f.restartErr +} + +var _ agentUpgradeExecutor = (*fakeAgentUpgradeExecutor)(nil) + type fakeMachineOperationStore struct { inProgress bool operation daemon.MachineOperation diff --git a/pkg/utils/utilexec/exec.go b/pkg/utils/utilexec/exec.go index 88f25478..a98d199f 100644 --- a/pkg/utils/utilexec/exec.go +++ b/pkg/utils/utilexec/exec.go @@ -140,6 +140,13 @@ func Systemctl() func(context.Context) *exec.Cmd { } } +// SystemdRun returns a command factory for systemd-run. +func SystemdRun() func(context.Context) *exec.Cmd { + return func(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "systemd-run") // #nosec G204 -- fixed binary + } +} + // Azcmagent returns a command factory for azcmagent. func Azcmagent() func(context.Context) *exec.Cmd { return func(ctx context.Context) *exec.Cmd { From 5cba8188db15996735c47ee4a83c2cf82c7de74c Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 21:56:27 +0000 Subject: [PATCH 02/45] refactor: reuse unbounded agent upgrade installer --- go.mod | 12 +- go.sum | 24 +-- pkg/daemon/agent_upgrade.go | 5 +- pkg/daemon/agent_upgrade_binary.go | 265 ++---------------------- pkg/daemon/agent_upgrade_binary_test.go | 262 +++-------------------- 5 files changed, 69 insertions(+), 499 deletions(-) diff --git a/go.mod b/go.mod index f7009d23..42fa31d0 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.2 + github.com/Azure/unbounded v0.2.3-0.20260806215326-5e0aab6faf4b github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 - k8s.io/api v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -101,12 +101,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.3 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index dd35efd4..48bb714c 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.2 h1:uu5hYj20UBbrSAlf4qnDiMTGaR0xxsYnkdbTWTdBHX8= -github.com/Azure/unbounded v0.2.2/go.mod h1:bZqzs6NIfXJqA8FRYSeajKJ0TYIqT8Xjfvfwu6OpHkw= +github.com/Azure/unbounded v0.2.3-0.20260806215326-5e0aab6faf4b h1:kCSMYUffaaeCRQiNoBgJ6x76kHPzYZIBJl5Y4qWKLvA= +github.com/Azure/unbounded v0.2.3-0.20260806215326-5e0aab6faf4b/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= @@ -348,14 +348,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= -k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= @@ -370,7 +370,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index f1410cf9..4c5aef95 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -56,13 +56,10 @@ func parseAgentUpgradeRequest(parameters map[string]string) (agentUpgradeRequest if request.downloadURL == "" { return agentUpgradeRequest{}, fmt.Errorf("missing required parameter %q", agentUpgradeDownloadURLParameter) } - if _, err := validateAgentUpgradeURL(request.downloadURL); err != nil { - return agentUpgradeRequest{}, err - } if request.sha256 == "" { return agentUpgradeRequest{}, fmt.Errorf("missing required parameter %q", agentUpgradeSHA256Parameter) } - if _, err := parseAgentUpgradeSHA256(request.sha256); err != nil { + if _, err := secureAgentInstallOptions(request.downloadURL, request.sha256); err != nil { return agentUpgradeRequest{}, err } return request, nil diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 687460df..a4e321df 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -1,43 +1,27 @@ package daemon import ( - "archive/tar" - "compress/gzip" "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" - "io" "log/slog" - "net/http" - "net/url" "os" - "os/exec" "path/filepath" "runtime" - "strings" "syscall" - "time" "github.com/Azure/AKSFlexNode/pkg/utils/utilio" + "github.com/Azure/unbounded/pkg/agent/agentbinary" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) const ( agentUpgradeBinaryMode = 0o755 agentUpgradeMaxArchiveBytes = 256 << 20 agentUpgradeMaxBinaryBytes = 256 << 20 - agentUpgradeVerifyTimeout = 30 * time.Second ) -type agentUpgradePaths struct { - BinaryPath string - BluePath string - GreenPath string - CurrentPath string - LastGoodPath string - SignalPath string -} +type agentUpgradePaths = goalstates.AgentUpgradePaths // ensureAgentUpgradeLayout migrates a legacy direct binary into the blue slot. // It is intentionally idempotent because bootstrap and daemon startup may both @@ -209,36 +193,6 @@ func replaceSymlink(linkPath, targetPath string) error { return os.Rename(tempPath, linkPath) } -func parseAgentUpgradeSHA256(value string) ([sha256.Size]byte, error) { - var expected [sha256.Size]byte - value = strings.TrimSpace(value) - value = strings.TrimPrefix(value, "sha256:") - decoded, err := hex.DecodeString(value) - if err != nil || len(decoded) != sha256.Size { - return expected, fmt.Errorf("expected SHA-256 must be exactly 64 hexadecimal characters") - } - copy(expected[:], decoded) - return expected, nil -} - -func validateAgentUpgradeURL(rawURL string) (*url.URL, error) { - parsed, err := url.ParseRequestURI(strings.TrimSpace(rawURL)) - if err != nil { - return nil, fmt.Errorf("invalid download URL") - } - if parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { - return nil, fmt.Errorf("download URL must be an HTTPS URL without user information") - } - return parsed, nil -} - -func redactedAgentUpgradeURL(parsed *url.URL) string { - redacted := *parsed - redacted.RawQuery = "" - redacted.Fragment = "" - return redacted.String() -} - func expectedAgentArchiveMember() (string, error) { switch runtime.GOARCH { case "amd64", "arm64": @@ -248,209 +202,30 @@ func expectedAgentArchiveMember() (string, error) { } } -// installAndSwitchAgentBinary downloads and verifies an archive before making -// either daemon symlink visible. The archive digest covers the compressed bytes. -func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, expectedDigest string, paths agentUpgradePaths) error { - return installAndSwitchAgentBinaryWithClient(ctx, log, newAgentUpgradeHTTPClient(), rawURL, expectedDigest, paths) -} - -func installAndSwitchAgentBinaryWithClient(ctx context.Context, log *slog.Logger, client *http.Client, rawURL, expectedDigest string, paths agentUpgradePaths) error { - parsedURL, err := validateAgentUpgradeURL(rawURL) - if err != nil { - return err - } - expected, err := parseAgentUpgradeSHA256(expectedDigest) - if err != nil { - return err - } +func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.SecureInstallOptions, error) { member, err := expectedAgentArchiveMember() if err != nil { - return err - } - currentTarget, err := resolvedExecutable(paths.CurrentPath) - if err != nil { - return fmt.Errorf("resolve current agent binary: %w", err) - } - targetPath := paths.BluePath - if currentTarget == paths.BluePath { - targetPath = paths.GreenPath - } - - archivePath, err := downloadAgentUpgradeArchive(ctx, client, parsedURL, expected) - if err != nil { - return err - } - defer os.Remove(archivePath) //nolint:errcheck // temporary archive cleanup - // The inactive slot may still be the target of last-good. Point last-good - // at the verified running binary before replacing that slot. - if err := replaceSymlink(paths.LastGoodPath, currentTarget); err != nil { - return fmt.Errorf("protect current agent as last-good: %w", err) - } - if err := extractAgentUpgradeBinary(archivePath, member, targetPath); err != nil { - return err - } - if err := verifyAgentBinary(ctx, targetPath); err != nil { - return err - } - if err := replaceSymlink(paths.CurrentPath, targetPath); err != nil { - return fmt.Errorf("update current agent symlink: %w", err) - } - log.Info("staged upgraded agent binary", "url", redactedAgentUpgradeURL(parsedURL), "previous", currentTarget, "current", targetPath) - return nil -} - -func newAgentUpgradeHTTPClient() *http.Client { - return &http.Client{ - Timeout: 10 * time.Minute, - CheckRedirect: func(req *http.Request, _ []*http.Request) error { - if req.URL.Scheme != "https" { - return fmt.Errorf("redirect to non-HTTPS URL is not allowed") - } - return nil - }, - } -} - -func downloadAgentUpgradeArchive(ctx context.Context, client *http.Client, parsedURL *url.URL, expected [sha256.Size]byte) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedURL.String(), http.NoBody) - if err != nil { - return "", fmt.Errorf("create agent archive request: %w", err) - } - resp, err := client.Do(req) - if err != nil { - if ctx.Err() != nil { - return "", fmt.Errorf("download agent archive from %s: %w", redactedAgentUpgradeURL(parsedURL), ctx.Err()) - } - // Redirect targets and transport errors may contain credential-bearing - // URLs, so do not propagate the transport's error text. - return "", fmt.Errorf("download agent archive from %s failed", redactedAgentUpgradeURL(parsedURL)) - } - defer resp.Body.Close() //nolint:errcheck // response body cleanup - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("download agent archive from %s: HTTP status %d", redactedAgentUpgradeURL(parsedURL), resp.StatusCode) - } - if resp.ContentLength > agentUpgradeMaxArchiveBytes { - return "", fmt.Errorf("agent archive exceeds %d-byte limit", agentUpgradeMaxArchiveBytes) - } - - temp, err := os.CreateTemp("", "aks-flex-node-upgrade-*.tar.gz") - if err != nil { - return "", fmt.Errorf("create temporary agent archive: %w", err) - } - path := temp.Name() - ok := false - defer func() { - _ = temp.Close() - if !ok { - _ = os.Remove(path) - } - }() - hasher := sha256.New() - limited := io.LimitReader(resp.Body, agentUpgradeMaxArchiveBytes+1) - n, err := io.Copy(io.MultiWriter(temp, hasher), limited) - if err != nil { - return "", fmt.Errorf("read agent archive: %w", err) + return agentbinary.SecureInstallOptions{}, err } - if n > agentUpgradeMaxArchiveBytes { - return "", fmt.Errorf("agent archive exceeds %d-byte limit", agentUpgradeMaxArchiveBytes) + opts := agentbinary.SecureInstallOptions{ + DownloadURL: rawURL, + ExpectedSHA256: expectedDigest, + ExpectedMember: member, + Mode: agentUpgradeBinaryMode, + MaxArchiveBytes: agentUpgradeMaxArchiveBytes, + MaxExtractedBytes: agentUpgradeMaxBinaryBytes, } - if !equalDigest(hasher.Sum(nil), expected[:]) { - return "", fmt.Errorf("agent archive SHA-256 does not match expected digest") + if err := agentbinary.ValidateSecureInstallOptions(opts); err != nil { + return agentbinary.SecureInstallOptions{}, err } - if err := temp.Close(); err != nil { - return "", fmt.Errorf("close temporary agent archive: %w", err) - } - ok = true - return path, nil -} - -func equalDigest(actual, expected []byte) bool { - if len(actual) != len(expected) { - return false - } - var different byte - for i := range actual { - different |= actual[i] ^ expected[i] - } - return different == 0 + return opts, nil } -func extractAgentUpgradeBinary(archivePath, expectedMember, targetPath string) (err error) { - archive, err := os.Open(archivePath) //nolint:gosec // path is an internally created temporary file - if err != nil { - return fmt.Errorf("open agent archive: %w", err) - } - defer func() { - if closeErr := archive.Close(); closeErr != nil && err == nil { - err = closeErr - } - }() - gz, err := gzip.NewReader(archive) +func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, expectedDigest string, paths agentUpgradePaths) error { + opts, err := secureAgentInstallOptions(rawURL, expectedDigest) if err != nil { - return fmt.Errorf("decompress agent archive: %w", err) - } - defer gz.Close() //nolint:errcheck // read errors are reported while extracting - - found := false - // Bound total decompressed input as well as the selected member so gzip - // bombs hidden in unrelated archive members cannot consume unbounded work. - decompressed := &countingReader{reader: io.LimitReader(gz, 2*agentUpgradeMaxBinaryBytes+1)} - tarReader := tar.NewReader(decompressed) - for { - header, nextErr := tarReader.Next() - if errors.Is(nextErr, io.EOF) { - break - } - if nextErr != nil { - return fmt.Errorf("read agent archive: %w", nextErr) - } - if header.Name == "" || filepath.IsAbs(header.Name) || filepath.Clean(header.Name) != header.Name || strings.Contains(header.Name, `\`) || strings.HasPrefix(header.Name, ".."+string(filepath.Separator)) { - return fmt.Errorf("agent archive contains unsafe member name %q", header.Name) - } - if header.Name != expectedMember { - return fmt.Errorf("agent archive contains unexpected member %q", header.Name) - } - if found { - return fmt.Errorf("agent archive contains duplicate member %q", expectedMember) - } - if header.Typeflag != tar.TypeReg || header.Size <= 0 || header.Size > agentUpgradeMaxBinaryBytes { - return fmt.Errorf("agent archive member %q is not a valid bounded regular file", expectedMember) - } - if err := utilio.InstallFileWithLimitedSize(targetPath, tarReader, agentUpgradeBinaryMode, agentUpgradeMaxBinaryBytes); err != nil { - return fmt.Errorf("install upgraded agent binary: %w", err) - } - found = true - } - if decompressed.count > 2*agentUpgradeMaxBinaryBytes { - return fmt.Errorf("decompressed agent archive exceeds %d-byte limit", 2*agentUpgradeMaxBinaryBytes) - } - if !found { - return fmt.Errorf("agent archive does not contain expected member %q", expectedMember) - } - return nil -} - -type countingReader struct { - reader io.Reader - count int64 -} - -func (r *countingReader) Read(data []byte) (int, error) { - n, err := r.reader.Read(data) - r.count += int64(n) - return n, err -} - -func verifyAgentBinary(ctx context.Context, path string) error { - verifyCtx, cancel := context.WithTimeout(ctx, agentUpgradeVerifyTimeout) - defer cancel() - cmd := exec.CommandContext(verifyCtx, path, "version") //nolint:gosec // fixed verified binary path and argument - // Candidate output is untrusted and could expose host data in operation - // status. Only the command's success is relevant to verification. - cmd.Stdout = io.Discard - cmd.Stderr = io.Discard - if err := cmd.Run(); err != nil { - return fmt.Errorf("verify upgraded agent binary: %w", err) + return err } - return nil + _, err = agentbinary.SecureInstallAndSwitch(ctx, log, paths, opts) + return err } diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go index c49e1861..fa55ed8b 100644 --- a/pkg/daemon/agent_upgrade_binary_test.go +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -1,17 +1,7 @@ package daemon import ( - "archive/tar" - "bytes" - "compress/gzip" - "context" - "crypto/sha256" - "fmt" - "io" "log/slog" - "net/http" - "net/http/httptest" - "net/url" "os" "path/filepath" "runtime" @@ -48,203 +38,48 @@ func TestEnsureAgentUpgradeLayoutMigratesLegacyBinaryIdempotently(t *testing.T) } } -func TestParseAgentUpgradeSHA256(t *testing.T) { +func TestSecureAgentInstallOptions(t *testing.T) { t.Parallel() - digest := strings.Repeat("ab", sha256.Size) - for _, value := range []string{digest, "sha256:" + digest} { - if _, err := parseAgentUpgradeSHA256(value); err != nil { - t.Fatalf("parseAgentUpgradeSHA256(%q): %v", value, err) - } - } - for _, value := range []string{"", "abc", strings.Repeat("z", 64)} { - if _, err := parseAgentUpgradeSHA256(value); err == nil { - t.Fatalf("parseAgentUpgradeSHA256(%q) error = nil", value) - } - } -} - -func TestValidateAgentUpgradeURL(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - value string - wantErr bool - }{ - "HTTPS": {value: "https://example.com/agent.tar.gz?sig=secret"}, - "HTTP": {value: "http://example.com/agent.tar.gz", wantErr: true}, - "file": {value: "file:///tmp/agent.tar.gz", wantErr: true}, - "userinfo": {value: "https://user:secret@example.com/agent.tar.gz", wantErr: true}, - "missing host": {value: "https:///agent.tar.gz", wantErr: true}, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - _, err := validateAgentUpgradeURL(tt.value) - if (err != nil) != tt.wantErr { - t.Fatalf("validateAgentUpgradeURL() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestRedactedAgentUpgradeURLRemovesQuery(t *testing.T) { - t.Parallel() - - parsed, err := url.Parse("https://example.com/agent.tar.gz?sig=secret") + digest := strings.Repeat("a", 64) + opts, err := secureAgentInstallOptions("https://example.com/agent.tar.gz?sig=secret", digest) if err != nil { - t.Fatalf("Parse: %v", err) - } - if got := redactedAgentUpgradeURL(parsed); got != "https://example.com/agent.tar.gz" { - t.Fatalf("redacted URL = %q", got) - } -} - -func TestDownloadAgentUpgradeArchiveVerifiesDigestAndRedactsErrors(t *testing.T) { - t.Parallel() - - payload := []byte("archive") - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write(payload) - })) - t.Cleanup(server.Close) - parsed, err := url.Parse(server.URL + "/agent.tar.gz?sig=secret") - if err != nil { - t.Fatalf("Parse: %v", err) - } - expected := sha256.Sum256(payload) - path, err := downloadAgentUpgradeArchive(t.Context(), server.Client(), parsed, expected) - if err != nil { - t.Fatalf("downloadAgentUpgradeArchive: %v", err) - } - t.Cleanup(func() { _ = os.Remove(path) }) - - wrong := sha256.Sum256([]byte("wrong")) - _, err = downloadAgentUpgradeArchive(t.Context(), server.Client(), parsed, wrong) - if err == nil { - t.Fatal("downloadAgentUpgradeArchive error = nil") - } - if strings.Contains(err.Error(), "secret") { - t.Fatalf("error leaked URL query: %v", err) - } -} - -func TestExtractAgentUpgradeBinary(t *testing.T) { - t.Parallel() - - member, err := expectedAgentArchiveMember() - if err != nil { - t.Skipf("unsupported test architecture: %v", err) - } - archivePath := filepath.Join(t.TempDir(), "agent.tar.gz") - binary := []byte("#!/bin/sh\nexit 0\n") - writeTestAgentArchive(t, archivePath, []testTarMember{{name: member, mode: 0o755, body: binary}}) - targetPath := filepath.Join(t.TempDir(), "agent") - if err := extractAgentUpgradeBinary(archivePath, member, targetPath); err != nil { - t.Fatalf("extractAgentUpgradeBinary: %v", err) - } - got, err := os.ReadFile(targetPath) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if !bytes.Equal(got, binary) { - t.Fatalf("binary = %q, want %q", got, binary) + t.Fatalf("secureAgentInstallOptions: %v", err) } - info, err := os.Stat(targetPath) - if err != nil { - t.Fatalf("Stat: %v", err) + if opts.ExpectedMember != "aks-flex-node-linux-"+runtime.GOARCH { + t.Fatalf("ExpectedMember = %q", opts.ExpectedMember) } - if info.Mode().Perm() != agentUpgradeBinaryMode { - t.Fatalf("mode = %o, want %o", info.Mode().Perm(), agentUpgradeBinaryMode) + if opts.MaxArchiveBytes != agentUpgradeMaxArchiveBytes || opts.MaxExtractedBytes != agentUpgradeMaxBinaryBytes { + t.Fatalf("size limits = %d, %d", opts.MaxArchiveBytes, opts.MaxExtractedBytes) } } -func TestExtractAgentUpgradeBinaryRejectsUnsafeAndUnexpectedArchives(t *testing.T) { +func TestSecureAgentInstallOptionsRejectsInvalidInputs(t *testing.T) { t.Parallel() - member := "aks-flex-node-linux-" + runtime.GOARCH - tests := map[string][]testTarMember{ - "missing member": {{name: "other", body: []byte("binary")}}, - "traversal": {{name: "../" + member, body: []byte("binary")}}, - "duplicate": { - {name: member, body: []byte("one")}, - {name: member, body: []byte("two")}, + tests := map[string]struct { + url string + digest string + }{ + "HTTP": { + url: "http://example.com/agent.tar.gz", + digest: strings.Repeat("a", 64), + }, + "invalid digest": { + url: "https://example.com/agent.tar.gz", + digest: "bad", }, } - for name, members := range tests { + for name, tt := range tests { t.Run(name, func(t *testing.T) { t.Parallel() - archivePath := filepath.Join(t.TempDir(), "agent.tar.gz") - writeTestAgentArchive(t, archivePath, members) - err := extractAgentUpgradeBinary(archivePath, member, filepath.Join(t.TempDir(), "agent")) - if err == nil { - t.Fatal("extractAgentUpgradeBinary error = nil") + if _, err := secureAgentInstallOptions(tt.url, tt.digest); err == nil { + t.Fatal("secureAgentInstallOptions error = nil") } }) } } -func TestInstallAndSwitchAgentBinarySwitchesAndProtectsLastGood(t *testing.T) { - t.Parallel() - - paths := testAgentUpgradePaths(t) - if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { - t.Fatalf("MkdirAll: %v", err) - } - oldBinary := []byte("#!/bin/sh\nexit 0\n") - if err := os.WriteFile(paths.BluePath, oldBinary, 0o755); err != nil { - t.Fatalf("WriteFile: %v", err) - } - if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { - t.Fatalf("Symlink current: %v", err) - } - if err := os.Symlink(paths.BluePath, paths.LastGoodPath); err != nil { - t.Fatalf("Symlink last-good: %v", err) - } - member, err := expectedAgentArchiveMember() - if err != nil { - t.Skipf("unsupported test architecture: %v", err) - } - goodArchive := filepath.Join(t.TempDir(), "good.tar.gz") - goodBinary := []byte("#!/bin/sh\nexit 0\n") - writeTestAgentArchive(t, goodArchive, []testTarMember{{name: member, body: goodBinary}}) - goodPayload, err := os.ReadFile(goodArchive) - if err != nil { - t.Fatalf("ReadFile good archive: %v", err) - } - badArchive := filepath.Join(t.TempDir(), "bad.tar.gz") - badBinary := []byte("#!/bin/sh\nexit 42\n") - writeTestAgentArchive(t, badArchive, []testTarMember{{name: member, body: badBinary}}) - badPayload, err := os.ReadFile(badArchive) - if err != nil { - t.Fatalf("ReadFile bad archive: %v", err) - } - - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - if request.URL.Path == "/bad.tar.gz" { - _, _ = w.Write(badPayload) - return - } - _, _ = w.Write(goodPayload) - })) - t.Cleanup(server.Close) - goodDigest := sha256.Sum256(goodPayload) - if err := installAndSwitchAgentBinaryWithClient(t.Context(), slog.Default(), server.Client(), server.URL+"/good.tar.gz", fmt.Sprintf("%x", goodDigest), paths); err != nil { - t.Fatalf("install good candidate: %v", err) - } - assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) - assertResolvedPath(t, paths.LastGoodPath, paths.BluePath) - - badDigest := sha256.Sum256(badPayload) - if err := installAndSwitchAgentBinaryWithClient(t.Context(), slog.Default(), server.Client(), server.URL+"/bad.tar.gz", fmt.Sprintf("%x", badDigest), paths); err == nil { - t.Fatal("install bad candidate error = nil") - } - assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) - // The failed candidate overwrote the inactive blue slot, so last-good must - // have moved to the still-running verified green slot first. - assertResolvedPath(t, paths.LastGoodPath, paths.GreenPath) -} - func TestInstallAndSwitchAgentBinaryRejectsInvalidInputsWithoutSwitching(t *testing.T) { t.Parallel() @@ -259,50 +94,19 @@ func TestInstallAndSwitchAgentBinaryRejectsInvalidInputsWithoutSwitching(t *test t.Fatalf("Symlink: %v", err) } - err := installAndSwitchAgentBinary(context.Background(), slog.Default(), "http://example.com/agent.tar.gz", strings.Repeat("0", 64), paths) + err := installAndSwitchAgentBinary( + t.Context(), + slog.Default(), + "http://example.com/agent.tar.gz", + strings.Repeat("0", 64), + paths, + ) if err == nil { t.Fatal("installAndSwitchAgentBinary error = nil") } assertResolvedPath(t, paths.CurrentPath, paths.BluePath) } -type testTarMember struct { - name string - mode int64 - body []byte -} - -func writeTestAgentArchive(t *testing.T, path string, members []testTarMember) { - t.Helper() - file, err := os.Create(path) //nolint:gosec // test-owned temporary path - if err != nil { - t.Fatalf("Create: %v", err) - } - gz := gzip.NewWriter(file) - tarWriter := tar.NewWriter(gz) - for _, member := range members { - mode := member.mode - if mode == 0 { - mode = 0o755 - } - if err := tarWriter.WriteHeader(&tar.Header{Name: member.name, Mode: mode, Size: int64(len(member.body)), Typeflag: tar.TypeReg}); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if _, err := io.Copy(tarWriter, bytes.NewReader(member.body)); err != nil { - t.Fatalf("Write: %v", err) - } - } - if err := tarWriter.Close(); err != nil { - t.Fatalf("close tar: %v", err) - } - if err := gz.Close(); err != nil { - t.Fatalf("close gzip: %v", err) - } - if err := file.Close(); err != nil { - t.Fatalf("close file: %v", err) - } -} - func testAgentUpgradePaths(t *testing.T) agentUpgradePaths { t.Helper() dir := t.TempDir() @@ -326,9 +130,3 @@ func assertResolvedPath(t *testing.T, path, want string) { t.Fatalf("resolved %s = %s, want %s", path, got, want) } } - -func Example_redactedAgentUpgradeURL() { - parsed, _ := url.Parse("https://example.com/agent.tar.gz?sig=secret") - fmt.Println(redactedAgentUpgradeURL(parsed)) - // Output: https://example.com/agent.tar.gz -} From 3448ea9fec469b381de3ed49ce417ede17e29c54 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 22:08:23 +0000 Subject: [PATCH 03/45] fix: converge upgrade recovery on daemon startup --- go.mod | 2 +- go.sum | 4 +- pkg/daemon/agent_upgrade_binary.go | 19 +++++++-- pkg/daemon/daemon.go | 5 +++ pkg/daemon/lifecycle.go | 56 +++++++++++++++++++------ pkg/daemon/lifecycle_test.go | 66 ++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 42fa31d0..f6b49f9f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260806215326-5e0aab6faf4b + github.com/Azure/unbounded v0.2.3-0.20260806220713-8173c7ee7a8c github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 48bb714c..50598160 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260806215326-5e0aab6faf4b h1:kCSMYUffaaeCRQiNoBgJ6x76kHPzYZIBJl5Y4qWKLvA= -github.com/Azure/unbounded v0.2.3-0.20260806215326-5e0aab6faf4b/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260806220713-8173c7ee7a8c h1:f+7hwOKLZaCOlBjTauidtR4Sq1m/Wd7hmkERvkgDviM= +github.com/Azure/unbounded v0.2.3-0.20260806220713-8173c7ee7a8c/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index a4e321df..122a5252 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/AKSFlexNode/pkg/utils/utilio" "github.com/Azure/unbounded/pkg/agent/agentbinary" - "github.com/Azure/unbounded/pkg/agent/goalstates" ) const ( @@ -21,7 +20,14 @@ const ( agentUpgradeMaxBinaryBytes = 256 << 20 ) -type agentUpgradePaths = goalstates.AgentUpgradePaths +type agentUpgradePaths struct { + BinaryPath string + BluePath string + GreenPath string + CurrentPath string + LastGoodPath string + SignalPath string +} // ensureAgentUpgradeLayout migrates a legacy direct binary into the blue slot. // It is intentionally idempotent because bootstrap and daemon startup may both @@ -226,6 +232,13 @@ func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, if err != nil { return err } - _, err = agentbinary.SecureInstallAndSwitch(ctx, log, paths, opts) + layout := agentbinary.Layout{ + BinaryPath: paths.BinaryPath, + BluePath: paths.BluePath, + GreenPath: paths.GreenPath, + CurrentPath: paths.CurrentPath, + LastGoodPath: paths.LastGoodPath, + } + _, err = agentbinary.SecureInstallAndSwitch(ctx, log, layout, opts) return err } diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index e1c891d1..79866867 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -31,6 +31,11 @@ const ( // Run starts the machine-driven daemon loop. func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { + // Existing direct-file installations may predate the recovery units. Keep + // the binary layout and systemd rollback assets converged on every startup. + if err := ensureAgentUpgradeServiceAssets(ctx, log); err != nil { + return err + } restCfg, stopCredentials, err := daemonRESTConfig(ctx, cfg) if err != nil { return err diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index d0b40698..d9b743da 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -1,6 +1,7 @@ package daemon import ( + "bytes" "context" _ "embed" "fmt" @@ -41,17 +42,54 @@ func InstallService(log *slog.Logger) phases.Task { func (t *installServiceTask) Name() string { return "install-service" } func (t *installServiceTask) Do(ctx context.Context) error { - if err := ensureAgentUpgradeLayout(ctx, t.log, defaultAgentUpgradePaths()); err != nil { + if err := ensureAgentUpgradeServiceAssets(ctx, t.log); err != nil { + return err + } + if err := utilexec.RunCmd(ctx, t.log, utilexec.Systemctl(), "enable", ServiceUnitName); err != nil { + return fmt.Errorf("systemctl enable %s: %w", ServiceUnitName, err) + } + if err := utilexec.RunCmd(ctx, t.log, utilexec.Systemctl(), "start", ServiceUnitName); err != nil { + return fmt.Errorf("systemctl start %s: %w", ServiceUnitName, err) + } + + t.log.Info("systemd service installed and started", "unit", ServiceUnitName) + return nil +} + +func ensureAgentUpgradeServiceAssets(ctx context.Context, log *slog.Logger) error { + return ensureAgentUpgradeServiceAssetsAt( + ctx, + log, + defaultAgentUpgradePaths(), + systemdSystemDir, + recoveryScriptPath, + utilexec.ReloadSystemd, + ) +} + +func ensureAgentUpgradeServiceAssetsAt( + ctx context.Context, + log *slog.Logger, + binaryPaths agentUpgradePaths, + systemdDir, recoveryScript string, + reload func(context.Context, *slog.Logger) error, +) error { + if err := ensureAgentUpgradeLayout(ctx, log, binaryPaths); err != nil { return fmt.Errorf("initialize agent binary layout: %w", err) } + recoveryServiceContent := bytes.ReplaceAll( + recoveryServiceUnitContent, + []byte(recoveryScriptPath), + []byte(recoveryScript), + ) assets := []struct { path string content []byte mode os.FileMode }{ - {path: filepath.Join(systemdSystemDir, ServiceUnitName), content: serviceUnitContent, mode: 0o644}, - {path: filepath.Join(systemdSystemDir, recoveryServiceUnitName), content: recoveryServiceUnitContent, mode: 0o644}, - {path: recoveryScriptPath, content: recoveryScriptContent, mode: 0o750}, + {path: filepath.Join(systemdDir, ServiceUnitName), content: serviceUnitContent, mode: 0o644}, + {path: filepath.Join(systemdDir, recoveryServiceUnitName), content: recoveryServiceContent, mode: 0o644}, + {path: recoveryScript, content: recoveryScriptContent, mode: 0o750}, } for _, asset := range assets { if err := utilio.WriteFile(asset.path, asset.content, asset.mode); err != nil { @@ -59,17 +97,9 @@ func (t *installServiceTask) Do(ctx context.Context) error { } } - if err := utilexec.ReloadSystemd(ctx, t.log); err != nil { + if err := reload(ctx, log); err != nil { return fmt.Errorf("systemctl daemon-reload: %w", err) } - if err := utilexec.RunCmd(ctx, t.log, utilexec.Systemctl(), "enable", ServiceUnitName); err != nil { - return fmt.Errorf("systemctl enable %s: %w", ServiceUnitName, err) - } - if err := utilexec.RunCmd(ctx, t.log, utilexec.Systemctl(), "start", ServiceUnitName); err != nil { - return fmt.Errorf("systemctl start %s: %w", ServiceUnitName, err) - } - - t.log.Info("systemd service installed and started", "unit", ServiceUnitName) return nil } diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go index c7053393..043e0d27 100644 --- a/pkg/daemon/lifecycle_test.go +++ b/pkg/daemon/lifecycle_test.go @@ -1,10 +1,76 @@ package daemon import ( + "context" + "log/slog" + "os" + "path/filepath" "strings" "testing" ) +func TestEnsureAgentUpgradeServiceAssetsMigratesExistingInstallation(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BinaryPath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.BinaryPath, []byte("legacy"), 0o755); err != nil { + t.Fatalf("write legacy binary: %v", err) + } + systemdDir := filepath.Join(t.TempDir(), "systemd") + if err := os.MkdirAll(systemdDir, 0o750); err != nil { + t.Fatalf("MkdirAll systemd: %v", err) + } + unitPath := filepath.Join(systemdDir, ServiceUnitName) + if err := os.WriteFile(unitPath, []byte("[Service]\nExecStart=/usr/local/bin/aks-flex-node agent\n"), 0o644); err != nil { + t.Fatalf("write legacy unit: %v", err) + } + recoveryPath := filepath.Join(t.TempDir(), "aks-flex-node-recovery.sh") + reloaded := false + if err := ensureAgentUpgradeServiceAssetsAt( + t.Context(), + slog.Default(), + paths, + systemdDir, + recoveryPath, + func(context.Context, *slog.Logger) error { + reloaded = true + return nil + }, + ); err != nil { + t.Fatalf("ensureAgentUpgradeServiceAssetsAt: %v", err) + } + if !reloaded { + t.Fatal("systemd reload was not requested") + } + assertResolvedPath(t, paths.BinaryPath, paths.BluePath) + assertResolvedPath(t, paths.CurrentPath, paths.BluePath) + assertResolvedPath(t, paths.LastGoodPath, paths.BluePath) + unit, err := os.ReadFile(unitPath) + if err != nil { + t.Fatalf("read service unit: %v", err) + } + if !strings.Contains(string(unit), "OnFailure="+recoveryServiceUnitName) { + t.Fatalf("updated unit does not include recovery: %s", unit) + } + recoveryService, err := os.ReadFile(filepath.Join(systemdDir, recoveryServiceUnitName)) + if err != nil { + t.Fatalf("recovery service was not installed: %v", err) + } + if !strings.Contains(string(recoveryService), "ExecStart="+recoveryPath) { + t.Fatalf("recovery service does not use installed script: %s", recoveryService) + } + info, err := os.Stat(recoveryPath) + if err != nil { + t.Fatalf("recovery script was not installed: %v", err) + } + if info.Mode().Perm() != 0o750 { + t.Fatalf("recovery script mode = %o, want 750", info.Mode().Perm()) + } +} + func TestAgentServiceIncludesUpgradeRecovery(t *testing.T) { t.Parallel() From 9fe1ad1050a8256baee096f35df035970327e788 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 22:37:28 +0000 Subject: [PATCH 04/45] chore: update shared agent upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f6b49f9f..357935aa 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260806220713-8173c7ee7a8c + github.com/Azure/unbounded v0.2.3-0.20260806223631-72a217985a70 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 50598160..8df78d6c 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260806220713-8173c7ee7a8c h1:f+7hwOKLZaCOlBjTauidtR4Sq1m/Wd7hmkERvkgDviM= -github.com/Azure/unbounded v0.2.3-0.20260806220713-8173c7ee7a8c/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260806223631-72a217985a70 h1:r2dBoPqb7AgeDKrH32qxcOHqkf3nbKCObYstTSowfOo= +github.com/Azure/unbounded v0.2.3-0.20260806223631-72a217985a70/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From f437ac92936940cc873019d983499d020f7a1370 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 22:43:34 +0000 Subject: [PATCH 05/45] chore: update shared upgrade implementation --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 357935aa..f5d90468 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260806223631-72a217985a70 + github.com/Azure/unbounded v0.2.3-0.20260806224251-75333be34462 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 8df78d6c..ce8010dd 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260806223631-72a217985a70 h1:r2dBoPqb7AgeDKrH32qxcOHqkf3nbKCObYstTSowfOo= -github.com/Azure/unbounded v0.2.3-0.20260806223631-72a217985a70/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260806224251-75333be34462 h1:8AmE8fDDUdOirM6CZX7oGBVGatl3i5jZQsFVpD+N1hY= +github.com/Azure/unbounded v0.2.3-0.20260806224251-75333be34462/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 0f3c10923424766be218551acefb3e781f1df64a Mon Sep 17 00:00:00 2001 From: Baichao He Date: Thu, 6 Aug 2026 22:51:23 +0000 Subject: [PATCH 06/45] chore: update reviewed upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f5d90468..3e1043e1 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260806224251-75333be34462 + github.com/Azure/unbounded v0.2.3-0.20260806225035-10ab536796c4 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index ce8010dd..378b52f0 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260806224251-75333be34462 h1:8AmE8fDDUdOirM6CZX7oGBVGatl3i5jZQsFVpD+N1hY= -github.com/Azure/unbounded v0.2.3-0.20260806224251-75333be34462/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260806225035-10ab536796c4 h1:sM2HXeaQRWxS2nksgYEk+AnbSQgz0YnDmjeYr3FbBmo= +github.com/Azure/unbounded v0.2.3-0.20260806225035-10ab536796c4/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 85da2b35f1bef4ae12ff33c1d4e8cfb06ba370eb Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 00:03:03 +0000 Subject: [PATCH 07/45] chore: update agent upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3e1043e1..efd28eb6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260806225035-10ab536796c4 + github.com/Azure/unbounded v0.2.3-0.20260807000213-d68bbe5c4445 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 378b52f0..725295be 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260806225035-10ab536796c4 h1:sM2HXeaQRWxS2nksgYEk+AnbSQgz0YnDmjeYr3FbBmo= -github.com/Azure/unbounded v0.2.3-0.20260806225035-10ab536796c4/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807000213-d68bbe5c4445 h1:sMPDX0LumaS3lK0z90NyPNC4Qv1am3xF1M+PzO6oDnQ= +github.com/Azure/unbounded v0.2.3-0.20260807000213-d68bbe5c4445/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 604ef213074c41dddd15dce0cd2204f8444386a8 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 00:13:03 +0000 Subject: [PATCH 08/45] chore: update agent upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index efd28eb6..0ef3336c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807000213-d68bbe5c4445 + github.com/Azure/unbounded v0.2.3-0.20260807001219-10efc812e817 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 725295be..e0b52878 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807000213-d68bbe5c4445 h1:sMPDX0LumaS3lK0z90NyPNC4Qv1am3xF1M+PzO6oDnQ= -github.com/Azure/unbounded v0.2.3-0.20260807000213-d68bbe5c4445/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807001219-10efc812e817 h1:HDSTLNqC+lhMexNZJ80+KmgIisdZ2RPchgqUffKXOV4= +github.com/Azure/unbounded v0.2.3-0.20260807001219-10efc812e817/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 2155e9ef5b70814e76e85ecc3f28201dbee3d103 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 00:21:33 +0000 Subject: [PATCH 09/45] chore: update agent upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0ef3336c..bd3b595d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807001219-10efc812e817 + github.com/Azure/unbounded v0.2.3-0.20260807002052-7e07b9ae8139 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index e0b52878..510bfb28 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807001219-10efc812e817 h1:HDSTLNqC+lhMexNZJ80+KmgIisdZ2RPchgqUffKXOV4= -github.com/Azure/unbounded v0.2.3-0.20260807001219-10efc812e817/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807002052-7e07b9ae8139 h1:pX0C2qIuWR6y7bsC+bh0menHm9B/N45ihUH5rcjf+No= +github.com/Azure/unbounded v0.2.3-0.20260807002052-7e07b9ae8139/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 4133c63d4f23395456a039806a5f231489aa6d90 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 00:32:38 +0000 Subject: [PATCH 10/45] chore: update agent upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bd3b595d..a5663961 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807002052-7e07b9ae8139 + github.com/Azure/unbounded v0.2.3-0.20260807003200-da53b6da31fb github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 510bfb28..5aa0cd32 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807002052-7e07b9ae8139 h1:pX0C2qIuWR6y7bsC+bh0menHm9B/N45ihUH5rcjf+No= -github.com/Azure/unbounded v0.2.3-0.20260807002052-7e07b9ae8139/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807003200-da53b6da31fb h1:J2mQKLkCcau6l2WIg3XhVK6kwsspEBVPNWRYbItZfpE= +github.com/Azure/unbounded v0.2.3-0.20260807003200-da53b6da31fb/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From d40f6589cd83b81fe8fea91de83a41d250d9081e Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 00:58:10 +0000 Subject: [PATCH 11/45] test: use absolute Flex Node binary path --- hack/e2e/lib/node-join.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e/lib/node-join.sh b/hack/e2e/lib/node-join.sh index 63db3c5c..35541411 100755 --- a/hack/e2e/lib/node-join.sh +++ b/hack/e2e/lib/node-join.sh @@ -44,7 +44,7 @@ sudo AKS_FLEX_NODE_LOCAL_BINARY=/tmp/aks-flex-node-binary \ SKIP_AZCLI=true \ bash /tmp/aks-flex-node-install.sh --yes -aks-flex-node version +/usr/local/bin/aks-flex-node version sudo cp /tmp/config.json /etc/aks-flex-node/ From 55a030ee2fd5e913cfd6256c0b56fe68215198cf Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 01:47:32 +0000 Subject: [PATCH 12/45] fix: install Flex Node binary with executable mode --- scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 5fc7a457..7007ae48 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -259,7 +259,7 @@ install_binary() { # Install binary cp "$binary_path" "$INSTALL_DIR/aks-flex-node" - chmod +x "$INSTALL_DIR/aks-flex-node" + chmod 0755 "$INSTALL_DIR/aks-flex-node" chown root:root "$INSTALL_DIR/aks-flex-node" log_success "Binary installed to $INSTALL_DIR/aks-flex-node" From e2c68c7b8f7a10706088748899b0bd9246b6594a Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 02:13:08 +0000 Subject: [PATCH 13/45] fix: install binary with explicit ownership and mode --- scripts/install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 7007ae48..3566535b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -257,10 +257,10 @@ install_binary() { log_info "Installing binary to $INSTALL_DIR..." - # Install binary - cp "$binary_path" "$INSTALL_DIR/aks-flex-node" - chmod 0755 "$INSTALL_DIR/aks-flex-node" - chown root:root "$INSTALL_DIR/aks-flex-node" + # Install with explicit ownership and modes so restrictive remote umasks do + # not leave the command inaccessible to non-root operators. + install -d -o root -g root -m 0755 "$INSTALL_DIR" + install -o root -g root -m 0755 "$binary_path" "$INSTALL_DIR/aks-flex-node" log_success "Binary installed to $INSTALL_DIR/aks-flex-node" } From dbd11f952411bd79cfe83d557767e86017a2daa6 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 02:34:03 +0000 Subject: [PATCH 14/45] test: reopen registry tunnel for image push --- hack/e2e/lib/controller.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/hack/e2e/lib/controller.sh b/hack/e2e/lib/controller.sh index 7f4b0d54..37c990e1 100644 --- a/hack/e2e/lib/controller.sh +++ b/hack/e2e/lib/controller.sh @@ -159,9 +159,6 @@ _build_controller_image() { log_section "Building AKS Flex Controller Image" log_info "Building controller image ${local_image} and pushing to in-cluster local registry" - pf_pid="" - _start_registry_port_forward pf_pid "${local_port}" || return 1 - if ! ( cd "${REPO_ROOT}" DOCKER_BUILDKIT=1 docker build \ @@ -172,13 +169,26 @@ _build_controller_image() { --build-arg "BUILD_TIME=${build_time}" \ --tag "${local_image}" \ . - docker push "${local_image}" ); then + return 1 + fi + + local pushed=0 attempt + for attempt in 1 2 3; do + pf_pid="" + if _start_registry_port_forward pf_pid "${local_port}" && docker push "${local_image}"; then + pushed=1 + _stop_registry_port_forward "${pf_pid}" + break + fi _stop_registry_port_forward "${pf_pid}" + log_warn "Controller image push attempt ${attempt} failed; reopening registry tunnel" + sleep 2 + done + if [[ "${pushed}" != "1" ]]; then return 1 fi - _stop_registry_port_forward "${pf_pid}" docker image rm "${local_image}" >/dev/null 2>&1 || true out_image="${cluster_image}" } From b750c2a2345ad867dcddd6b78fd4fb225c5ad421 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 02:59:43 +0000 Subject: [PATCH 15/45] test: validate AgentUpgrade slot transition --- hack/e2e/lib/agent-upgrade.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hack/e2e/lib/agent-upgrade.sh b/hack/e2e/lib/agent-upgrade.sh index 27e83256..a5416763 100644 --- a/hack/e2e/lib/agent-upgrade.sh +++ b/hack/e2e/lib/agent-upgrade.sh @@ -179,7 +179,7 @@ REMOTE agent_upgrade_e2e() { log_section "Managed AgentUpgrade E2E" - local vm_name vm_ip suffix success_digest failure_digest before before_binary_digest success_snapshot success_binary_digest rollback_snapshot rollback_binary_digest retry_snapshot retry_binary_digest + local vm_name vm_ip suffix success_digest failure_digest before before_slot success_snapshot success_slot success_binary_digest rollback_snapshot rollback_binary_digest retry_snapshot retry_binary_digest vm_name="$(state_get token_vm_name)" vm_ip="$(state_get token_vm_ip)" suffix="$(date +%s)" @@ -190,7 +190,7 @@ agent_upgrade_e2e() { success_digest="$(_agent_upgrade_digest "${vm_ip}" success.tar.gz)" failure_digest="$(_agent_upgrade_digest "${vm_ip}" failure.tar.gz)" before="$(_agent_upgrade_snapshot "${vm_ip}")" - before_binary_digest="$(cut -d'|' -f3 <<<"${before}")" + before_slot="$(cut -d'|' -f1 <<<"${before}")" log_info "Pre-upgrade agent snapshot: ${before}" local success_op="agent-upgrade-success-${suffix}" @@ -199,9 +199,9 @@ agent_upgrade_e2e() { validate_node_joined "${vm_name}" _agent_upgrade_assert_synchronized "${vm_ip}" success_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" - IFS='|' read -r _ _ success_binary_digest _ <<<"${success_snapshot}" - if [[ -z "${success_binary_digest}" || "${success_binary_digest}" == "${before_binary_digest}" ]]; then - log_error "Successful AgentUpgrade did not replace the running binary: before=${before} after=${success_snapshot}" + IFS='|' read -r success_slot _ success_binary_digest _ <<<"${success_snapshot}" + if [[ -z "${success_slot}" || "${success_slot}" == "${before_slot}" ]]; then + log_error "Successful AgentUpgrade did not switch the active binary slot: before=${before} after=${success_snapshot}" return 1 fi From 4d21281dbaa2916e6c9528cf92c68be168f0e95d Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 18:55:56 +0000 Subject: [PATCH 16/45] test: probe installed agent as root --- hack/e2e/lib/node-join.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e/lib/node-join.sh b/hack/e2e/lib/node-join.sh index 35541411..e92499a9 100755 --- a/hack/e2e/lib/node-join.sh +++ b/hack/e2e/lib/node-join.sh @@ -44,7 +44,7 @@ sudo AKS_FLEX_NODE_LOCAL_BINARY=/tmp/aks-flex-node-binary \ SKIP_AZCLI=true \ bash /tmp/aks-flex-node-install.sh --yes -/usr/local/bin/aks-flex-node version +sudo /usr/local/bin/aks-flex-node version sudo cp /tmp/config.json /etc/aks-flex-node/ From b1bf66208561021225c62b6556c9201cc25d457a Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 19:26:28 +0000 Subject: [PATCH 17/45] refactor: adapt to latest shared upgrade API --- go.mod | 2 +- go.sum | 4 +-- pkg/daemon/agent_upgrade_binary.go | 33 ++++++++++++++++++++----- pkg/daemon/agent_upgrade_binary_test.go | 11 +++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index a5663961..eaab347f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807003200-da53b6da31fb + github.com/Azure/unbounded v0.2.3-0.20260807192210-075694a4c521 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 5aa0cd32..67de5fd7 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807003200-da53b6da31fb h1:J2mQKLkCcau6l2WIg3XhVK6kwsspEBVPNWRYbItZfpE= -github.com/Azure/unbounded v0.2.3-0.20260807003200-da53b6da31fb/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807192210-075694a4c521 h1:zwJR+JAyMc3xyKUtA+GTwVqQIgjFtkWzLrOMjgqY8Q4= +github.com/Azure/unbounded v0.2.3-0.20260807192210-075694a4c521/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 122a5252..1b3c918f 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -5,10 +5,14 @@ import ( "errors" "fmt" "log/slog" + "net/http" + "net/url" "os" "path/filepath" "runtime" + "strings" "syscall" + "time" "github.com/Azure/AKSFlexNode/pkg/utils/utilio" "github.com/Azure/unbounded/pkg/agent/agentbinary" @@ -208,21 +212,38 @@ func expectedAgentArchiveMember() (string, error) { } } -func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.SecureInstallOptions, error) { +func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.InstallOptions, error) { + parsedURL, err := url.ParseRequestURI(strings.TrimSpace(rawURL)) + if err != nil || parsedURL.Scheme != "https" || parsedURL.Host == "" || parsedURL.User != nil || parsedURL.Fragment != "" { + return agentbinary.InstallOptions{}, fmt.Errorf("download URL must use HTTPS, include a host, omit user information, and omit fragments") + } member, err := expectedAgentArchiveMember() if err != nil { - return agentbinary.SecureInstallOptions{}, err + return agentbinary.InstallOptions{}, err } - opts := agentbinary.SecureInstallOptions{ + opts := agentbinary.InstallOptions{ DownloadURL: rawURL, ExpectedSHA256: expectedDigest, ExpectedMember: member, Mode: agentUpgradeBinaryMode, MaxArchiveBytes: agentUpgradeMaxArchiveBytes, MaxExtractedBytes: agentUpgradeMaxBinaryBytes, + ExactMember: true, + HTTPClient: &http.Client{ + Timeout: 10 * time.Minute, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if req.URL.Scheme != "https" { + return fmt.Errorf("redirect to non-HTTPS URL is not allowed") + } + if len(via) >= 10 { + return fmt.Errorf("stopped after 10 redirects") + } + return nil + }, + }, } - if err := agentbinary.ValidateSecureInstallOptions(opts); err != nil { - return agentbinary.SecureInstallOptions{}, err + if err := agentbinary.ValidateInstallOptions(opts); err != nil { + return agentbinary.InstallOptions{}, err } return opts, nil } @@ -239,6 +260,6 @@ func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, CurrentPath: paths.CurrentPath, LastGoodPath: paths.LastGoodPath, } - _, err = agentbinary.SecureInstallAndSwitch(ctx, log, layout, opts) + _, err = agentbinary.InstallAndSwitchFromTarGzWithOptions(ctx, log, layout, opts) return err } diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go index fa55ed8b..6a4a7a1e 100644 --- a/pkg/daemon/agent_upgrade_binary_test.go +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -2,6 +2,7 @@ package daemon import ( "log/slog" + "net/http" "os" "path/filepath" "runtime" @@ -52,6 +53,16 @@ func TestSecureAgentInstallOptions(t *testing.T) { if opts.MaxArchiveBytes != agentUpgradeMaxArchiveBytes || opts.MaxExtractedBytes != agentUpgradeMaxBinaryBytes { t.Fatalf("size limits = %d, %d", opts.MaxArchiveBytes, opts.MaxExtractedBytes) } + if !opts.ExactMember { + t.Fatal("ExactMember = false") + } + redirect, err := http.NewRequest(http.MethodGet, "http://example.com/agent.tar.gz", http.NoBody) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + if err := opts.HTTPClient.CheckRedirect(redirect, nil); err == nil { + t.Fatal("HTTP redirect was accepted") + } } func TestSecureAgentInstallOptionsRejectsInvalidInputs(t *testing.T) { From f45bf6e7a0499d42b96525e4b6d2b9e2822072e2 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 19:28:43 +0000 Subject: [PATCH 18/45] chore: update Unbounded after main merge --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eaab347f..90ee6d70 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807192210-075694a4c521 + github.com/Azure/unbounded v0.2.3-0.20260807192706-4b46377f68cc github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 67de5fd7..4407be94 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807192210-075694a4c521 h1:zwJR+JAyMc3xyKUtA+GTwVqQIgjFtkWzLrOMjgqY8Q4= -github.com/Azure/unbounded v0.2.3-0.20260807192210-075694a4c521/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807192706-4b46377f68cc h1:X2XWHz5bfmd6X8qeKp5yGy7onxRgBSUj0hP8pUxA7xw= +github.com/Azure/unbounded v0.2.3-0.20260807192706-4b46377f68cc/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 0681fa88c0574926005836f3984f3a089df9d28e Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 20:11:36 +0000 Subject: [PATCH 19/45] refactor: follow simplified Unbounded upgrade API --- go.mod | 2 +- go.sum | 4 ++-- pkg/daemon/agent_upgrade_binary.go | 12 ++++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 90ee6d70..acc1e07c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807192706-4b46377f68cc + github.com/Azure/unbounded v0.2.3-0.20260807201022-1d82814a2b8e github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 4407be94..2afc9c0b 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807192706-4b46377f68cc h1:X2XWHz5bfmd6X8qeKp5yGy7onxRgBSUj0hP8pUxA7xw= -github.com/Azure/unbounded v0.2.3-0.20260807192706-4b46377f68cc/go.mod h1:LFzyTjRP4xwLSq7LDfKYDLhMyTPajPouwqxpc9tAy+M= +github.com/Azure/unbounded v0.2.3-0.20260807201022-1d82814a2b8e h1:PVFa5rJV1G4+H4mJPBUUGN7q2QD7ZOnshdv3og4UvgQ= +github.com/Azure/unbounded v0.2.3-0.20260807201022-1d82814a2b8e/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 1b3c918f..888b0f42 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -2,6 +2,8 @@ package daemon import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "log/slog" @@ -217,6 +219,11 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta if err != nil || parsedURL.Scheme != "https" || parsedURL.Host == "" || parsedURL.User != nil || parsedURL.Fragment != "" { return agentbinary.InstallOptions{}, fmt.Errorf("download URL must use HTTPS, include a host, omit user information, and omit fragments") } + digest := strings.TrimPrefix(strings.TrimSpace(expectedDigest), "sha256:") + decodedDigest, err := hex.DecodeString(digest) + if err != nil || len(decodedDigest) != sha256.Size { + return agentbinary.InstallOptions{}, fmt.Errorf("expected SHA-256 must be exactly 64 hexadecimal characters") + } member, err := expectedAgentArchiveMember() if err != nil { return agentbinary.InstallOptions{}, err @@ -242,9 +249,6 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta }, }, } - if err := agentbinary.ValidateInstallOptions(opts); err != nil { - return agentbinary.InstallOptions{}, err - } return opts, nil } @@ -260,6 +264,6 @@ func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, CurrentPath: paths.CurrentPath, LastGoodPath: paths.LastGoodPath, } - _, err = agentbinary.InstallAndSwitchFromTarGzWithOptions(ctx, log, layout, opts) + _, err = agentbinary.InstallAndSwitchFromTarGz(ctx, log, layout, opts) return err } From a62f69248f3d8bbd68ec3077ef276bf6a4f60662 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 20:32:42 +0000 Subject: [PATCH 20/45] chore: bump Unbounded upgrade branch --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index acc1e07c..ea020358 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807201022-1d82814a2b8e + github.com/Azure/unbounded v0.2.3-0.20260807203116-427e5341eaa5 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 2afc9c0b..5de41fa5 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807201022-1d82814a2b8e h1:PVFa5rJV1G4+H4mJPBUUGN7q2QD7ZOnshdv3og4UvgQ= -github.com/Azure/unbounded v0.2.3-0.20260807201022-1d82814a2b8e/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= +github.com/Azure/unbounded v0.2.3-0.20260807203116-427e5341eaa5 h1:MOWvC9Ch39PHDI7iDnVBdd451xF8UW04WYcHgWhBpnM= +github.com/Azure/unbounded v0.2.3-0.20260807203116-427e5341eaa5/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From ad14c6e1aa5870f5d1a2b92fbcda1bab23d7a83b Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 7 Aug 2026 20:34:13 +0000 Subject: [PATCH 21/45] chore: update Unbounded upgrade dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ea020358..5c0f1625 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260807203116-427e5341eaa5 + github.com/Azure/unbounded v0.2.3-0.20260807203328-3d9e93b1b8a3 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 5de41fa5..d92dd199 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260807203116-427e5341eaa5 h1:MOWvC9Ch39PHDI7iDnVBdd451xF8UW04WYcHgWhBpnM= -github.com/Azure/unbounded v0.2.3-0.20260807203116-427e5341eaa5/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= +github.com/Azure/unbounded v0.2.3-0.20260807203328-3d9e93b1b8a3 h1:ynYXrziNDg3b/kJ4pw13ukF1dXUlGW2m3dCoOiCeZO0= +github.com/Azure/unbounded v0.2.3-0.20260807203328-3d9e93b1b8a3/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 411b5e5fca9021344ccf3542a7fb98820105b872 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 17:28:26 +0000 Subject: [PATCH 22/45] refactor: reuse shared host agent activation --- cmd/aks-flex-node/main.go | 1 + docs/usages/operations.md | 11 +- go.mod | 2 +- go.sum | 4 +- pkg/cmd/daemon/host_agent_upgrade.go | 58 ++++++ pkg/daemon/agent_upgrade.go | 6 + pkg/daemon/agent_upgrade_binary.go | 104 +++------- pkg/daemon/host_agent_activation.go | 183 ++++++++++++++++++ pkg/daemon/host_agent_activation_test.go | 56 ++++++ pkg/daemon/lifecycle.go | 49 +++-- pkg/daemon/lifecycle_test.go | 3 + pkg/daemon/machineoperation_reconciler.go | 14 ++ .../machineoperation_reconciler_test.go | 32 +++ 13 files changed, 429 insertions(+), 94 deletions(-) create mode 100644 pkg/cmd/daemon/host_agent_upgrade.go create mode 100644 pkg/daemon/host_agent_activation.go create mode 100644 pkg/daemon/host_agent_activation_test.go diff --git a/cmd/aks-flex-node/main.go b/cmd/aks-flex-node/main.go index eb071ee9..2dc07885 100644 --- a/cmd/aks-flex-node/main.go +++ b/cmd/aks-flex-node/main.go @@ -30,6 +30,7 @@ func main() { rootCmd.AddCommand(bootstrapdata.NewCommand()) rootCmd.AddCommand(preflight.NewCommand()) rootCmd.AddCommand(daemon.NewCommand()) + rootCmd.AddCommand(daemon.NewHostAgentUpgradeCommand()) rootCmd.AddCommand(daemon.NewAgentUpgradeRecoveryCommand()) rootCmd.AddCommand(reset.NewCommand()) rootCmd.AddCommand(version.NewCommand()) diff --git a/docs/usages/operations.md b/docs/usages/operations.md index 26057c20..84b51bf2 100644 --- a/docs/usages/operations.md +++ b/docs/usages/operations.md @@ -49,7 +49,7 @@ journalctl -u aks-flex-node-agent -f ## Managed Agent Upgrade -When the Machina `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTPS release archive and the SHA-256 of the compressed archive: +When the Unbounded `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTPS release archive and the SHA-256 of the compressed archive: ```yaml apiVersion: unbounded-cloud.io/v1alpha3 @@ -74,6 +74,15 @@ MachineOperations are cluster-scoped. The daemon group requires cluster-wide rea kubectl get machineoperation upgrade-agent-worker-01 -w ``` +A host provisioning system that has already authenticated and staged a candidate can activate it directly without creating an Unbounded `MachineOperation`: + +```bash +/var/tmp/aks-flex-node-candidate agent-upgrade --preflight +sudo /var/tmp/aks-flex-node-candidate agent-upgrade +``` + +The candidate must be staged separately from the installed binary. Direct activation and `MachineOperation` activation share one host lock and refuse to overlap with a pending operation signal. Both paths verify the candidate, switch the same blue/green layout, restart `aks-flex-node-agent.service`, verify the running executable, synchronize the active nspawn exec-credential binary, and restore last-good on activation failure. + ## Nspawn Worker Inspect the local nspawn-backed worker: diff --git a/go.mod b/go.mod index 657f7f39..797efc58 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260810063604-6442a3cd1398 + github.com/Azure/unbounded v0.2.3-0.20260810172605-4ad184157a14 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 70647a57..dd9f8089 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260810063604-6442a3cd1398 h1:4WhezGhXtgjgwUOg0lPg9btDUlPh6aTIkND2L5gs1Eg= -github.com/Azure/unbounded v0.2.3-0.20260810063604-6442a3cd1398/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= +github.com/Azure/unbounded v0.2.3-0.20260810172605-4ad184157a14 h1:APWKYrmdrkCq3uZAnRvNFuFvOY+Z8IKEIEjnKxKEIro= +github.com/Azure/unbounded v0.2.3-0.20260810172605-4ad184157a14/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= diff --git a/pkg/cmd/daemon/host_agent_upgrade.go b/pkg/cmd/daemon/host_agent_upgrade.go new file mode 100644 index 00000000..fc8e47fe --- /dev/null +++ b/pkg/cmd/daemon/host_agent_upgrade.go @@ -0,0 +1,58 @@ +package daemon + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + hostdaemon "github.com/Azure/AKSFlexNode/pkg/daemon" + "github.com/Azure/AKSFlexNode/pkg/logger" +) + +// NewHostAgentUpgradeCommand returns the hidden host-driven activation command. +func NewHostAgentUpgradeCommand() *cobra.Command { + var preflight bool + cmd := &cobra.Command{ + Use: "agent-upgrade", + Short: "Activate this executable as the host agent daemon", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + candidate, err := os.Executable() + if err != nil { + return fmt.Errorf("resolve candidate executable: %w", err) + } + candidate, err = filepath.Abs(candidate) + if err != nil { + return fmt.Errorf("resolve absolute candidate executable path: %w", err) + } + log := logger.CreateLogger("info", "") + if preflight { + plan, err := hostdaemon.PreflightHostAgentActivation(cmd.Context(), log, filepath.Clean(candidate)) + if err != nil { + return err + } + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Candidate: %s\nActive binary: %s\nInstall target: %s\n", plan.CandidatePath, plan.ActivePath, plan.TargetPath); err != nil { + return err + } + for _, action := range plan.Actions { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "- %s\n", action); err != nil { + return err + } + } + _, err = fmt.Fprintln(cmd.OutOrStdout(), "Preflight: no changes applied") + return err + } + result, err := hostdaemon.ActivateHostAgent(cmd.Context(), log, filepath.Clean(candidate)) + if err != nil { + return err + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "activated host agent daemon: %s -> %s\n", result.PreviousPath, result.CurrentPath) + return err + }, + } + cmd.Flags().BoolVar(&preflight, "preflight", false, "Show and validate the host activation plan without applying it") + return cmd +} diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 4c5aef95..1c887adb 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/AKSFlexNode/pkg/utils/utilexec" "github.com/Azure/AKSFlexNode/pkg/utils/utilio" machinav1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/pkg/agent/agentbinary" agentdaemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -159,6 +160,7 @@ func (s agentUpgradeSignalStore) clear() error { } type agentUpgradeExecutor interface { + Acquire() (io.Closer, error) RecordPending(context.Context, string) error RecordFailure(string) error Stage(context.Context, agentUpgradeRequest) error @@ -209,6 +211,10 @@ func newDaemonInstanceID() (string, error) { return hex.EncodeToString(value[:]), nil } +func (e *hostAgentUpgradeExecutor) Acquire() (io.Closer, error) { + return agentbinary.AcquireHostActivationLock(agentUpgradeLockPath) +} + func (e *hostAgentUpgradeExecutor) RecordPending(ctx context.Context, operationName string) error { existing, err := e.signals.read() if err != nil { diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 888b0f42..30e20901 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -18,12 +18,14 @@ import ( "github.com/Azure/AKSFlexNode/pkg/utils/utilio" "github.com/Azure/unbounded/pkg/agent/agentbinary" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) const ( agentUpgradeBinaryMode = 0o755 agentUpgradeMaxArchiveBytes = 256 << 20 agentUpgradeMaxBinaryBytes = 256 << 20 + agentUpgradeLockPath = "/run/aks-flex-node-agent-upgrade.lock" ) type agentUpgradePaths struct { @@ -35,13 +37,30 @@ type agentUpgradePaths struct { SignalPath string } -// ensureAgentUpgradeLayout migrates a legacy direct binary into the blue slot. -// It is intentionally idempotent because bootstrap and daemon startup may both -// call it while converging an older installation. -func ensureAgentUpgradeLayout(ctx context.Context, log *slog.Logger, paths agentUpgradePaths) error { - if err := ctx.Err(); err != nil { - return err +func (p agentUpgradePaths) layout() agentbinary.Layout { + return agentbinary.Layout{ + BinaryPath: p.BinaryPath, + BluePath: p.BluePath, + GreenPath: p.GreenPath, + CurrentPath: p.CurrentPath, + LastGoodPath: p.LastGoodPath, + } +} + +func (p agentUpgradePaths) sharedPaths() goalstates.AgentUpgradePaths { + return goalstates.AgentUpgradePaths{ + BinaryPath: p.BinaryPath, + BluePath: p.BluePath, + GreenPath: p.GreenPath, + CurrentPath: p.CurrentPath, + LastGoodPath: p.LastGoodPath, + SignalPath: p.SignalPath, } +} + +// ensureAgentUpgradeLayout adds Flex-specific ownership validation around the +// shared idempotent migration and link initialization implementation. +func ensureAgentUpgradeLayout(ctx context.Context, log *slog.Logger, paths agentUpgradePaths) error { if log == nil { return fmt.Errorf("logger is nil") } @@ -52,56 +71,12 @@ func ensureAgentUpgradeLayout(ctx context.Context, log *slog.Logger, paths agent if productionPaths && os.Geteuid() != 0 { return fmt.Errorf("agent binary layout must be initialized as root") } - if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { - return fmt.Errorf("create agent binary slot directory: %w", err) - } - - currentTarget, err := resolvedExecutable(paths.CurrentPath) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("resolve current agent binary: %w", err) - } - if errors.Is(err, os.ErrNotExist) { - seed, seedErr := initialAgentBinary(paths) - if seedErr != nil { - return fmt.Errorf("find initial agent binary: %w", seedErr) - } - if seed == paths.BinaryPath { - if err := copyExecutable(paths.BinaryPath, paths.BluePath); err != nil { - return fmt.Errorf("migrate legacy agent binary: %w", err) - } - seed = paths.BluePath - } - if err := replaceSymlink(paths.CurrentPath, seed); err != nil { - return fmt.Errorf("initialize current agent symlink: %w", err) - } - currentTarget = seed - } - - if _, err := resolvedExecutable(paths.LastGoodPath); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("resolve last-good agent binary: %w", err) - } - if err := replaceSymlink(paths.LastGoodPath, currentTarget); err != nil { - return fmt.Errorf("initialize last-good agent symlink: %w", err) - } - } - - binaryTarget, err := filepath.EvalSymlinks(paths.BinaryPath) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("resolve compatibility agent binary: %w", err) - } - if errors.Is(err, os.ErrNotExist) || binaryTarget != currentTarget { - if err := replaceSymlink(paths.BinaryPath, paths.CurrentPath); err != nil { - return fmt.Errorf("initialize compatibility agent symlink: %w", err) - } + if err := agentbinary.EnsureDaemonBinaryLinks(ctx, log, paths.sharedPaths()); err != nil { + return err } - if productionPaths { - if err := validateRootOwnedAgentUpgradePaths(paths); err != nil { - return err - } + return validateRootOwnedAgentUpgradePaths(paths) } - log.Info("agent binary blue-green layout initialized", "current", paths.CurrentPath, "last_good", paths.LastGoodPath) return nil } @@ -145,15 +120,6 @@ func validateAgentUpgradePaths(paths agentUpgradePaths) error { return nil } -func initialAgentBinary(paths agentUpgradePaths) (string, error) { - for _, path := range []string{paths.BluePath, paths.GreenPath, paths.BinaryPath} { - if _, err := resolvedExecutable(path); err == nil { - return path, nil - } - } - return "", os.ErrNotExist -} - func resolvedExecutable(path string) (string, error) { resolved, err := filepath.EvalSymlinks(path) if err != nil { @@ -228,7 +194,7 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta if err != nil { return agentbinary.InstallOptions{}, err } - opts := agentbinary.InstallOptions{ + return agentbinary.InstallOptions{ DownloadURL: rawURL, ExpectedSHA256: expectedDigest, ExpectedMember: member, @@ -248,8 +214,7 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta return nil }, }, - } - return opts, nil + }, nil } func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, expectedDigest string, paths agentUpgradePaths) error { @@ -257,13 +222,6 @@ func installAndSwitchAgentBinary(ctx context.Context, log *slog.Logger, rawURL, if err != nil { return err } - layout := agentbinary.Layout{ - BinaryPath: paths.BinaryPath, - BluePath: paths.BluePath, - GreenPath: paths.GreenPath, - CurrentPath: paths.CurrentPath, - LastGoodPath: paths.LastGoodPath, - } - _, err = agentbinary.InstallAndSwitchFromTarGz(ctx, log, layout, opts) + _, err = agentbinary.InstallAndSwitchFromTarGz(ctx, log, paths.layout(), opts) return err } diff --git a/pkg/daemon/host_agent_activation.go b/pkg/daemon/host_agent_activation.go new file mode 100644 index 00000000..0fcdaef6 --- /dev/null +++ b/pkg/daemon/host_agent_activation.go @@ -0,0 +1,183 @@ +package daemon + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/Azure/AKSFlexNode/pkg/utils/utilexec" + "github.com/Azure/unbounded/pkg/agent/agentbinary" +) + +const ( + hostAgentHealthTimeout = 30 * time.Second + hostAgentStableDuration = 3 * time.Second + hostAgentHealthPoll = 250 * time.Millisecond +) + +// PreflightHostAgentActivation validates a directly staged Flex agent binary +// and returns the shared activation plan without changing host state. +func PreflightHostAgentActivation(ctx context.Context, log *slog.Logger, candidatePath string) (agentbinary.ActivationPlan, error) { + service, paths, err := newFlexDaemonActivationService(log) + if err != nil { + return agentbinary.ActivationPlan{}, err + } + return agentbinary.PreflightHostDaemonActivation(ctx, hostAgentActivationOptions(paths, candidatePath), service) +} + +// ActivateHostAgent activates a directly staged Flex agent binary. It uses the +// same lock and binary layout as MachineOperation upgrades. +func ActivateHostAgent(ctx context.Context, log *slog.Logger, candidatePath string) (agentbinary.ActivationResult, error) { + if os.Geteuid() != 0 { + return agentbinary.ActivationResult{}, fmt.Errorf("host agent upgrade requires root privileges") + } + service, paths, err := newFlexDaemonActivationService(log) + if err != nil { + return agentbinary.ActivationResult{}, err + } + return agentbinary.ActivateHostDaemon(ctx, log, hostAgentActivationOptions(paths, candidatePath), service) +} + +func hostAgentActivationOptions(paths agentUpgradePaths, candidatePath string) agentbinary.ActivationOptions { + return agentbinary.ActivationOptions{ + Layout: paths.layout(), + CandidatePath: candidatePath, + BinaryMode: agentUpgradeBinaryMode, + LockPath: agentUpgradeLockPath, + } +} + +type flexDaemonActivationService struct { + log *slog.Logger + paths agentUpgradePaths + state stateStore + systemdDir string + recoveryScript string +} + +func newFlexDaemonActivationService(log *slog.Logger) (*flexDaemonActivationService, agentUpgradePaths, error) { + if log == nil { + log = slog.Default() + } + state, err := NewFileStateStore() + if err != nil { + return nil, agentUpgradePaths{}, err + } + paths := defaultAgentUpgradePaths() + return &flexDaemonActivationService{ + log: log, + paths: paths, + state: state, + systemdDir: systemdSystemDir, + recoveryScript: recoveryScriptPath, + }, paths, nil +} + +func (s *flexDaemonActivationService) Preflight(_ context.Context, currentBinaryPath string) (agentbinary.ServicePlan, error) { + if _, err := os.Stat(s.paths.SignalPath); err == nil { + return agentbinary.ServicePlan{}, fmt.Errorf("AgentUpgrade MachineOperation signal exists at %s", s.paths.SignalPath) + } else if !errors.Is(err, os.ErrNotExist) { + return agentbinary.ServicePlan{}, fmt.Errorf("inspect AgentUpgrade MachineOperation signal: %w", err) + } + for _, asset := range desiredAgentServiceAssets(s.paths, s.systemdDir, s.recoveryScript, currentBinaryPath) { + actual, err := os.ReadFile(asset.path) + if errors.Is(err, os.ErrNotExist) || err == nil && !bytes.Equal(actual, asset.content) { + return agentbinary.ServicePlan{ + UpdateRequired: true, + Description: "install or update AKS Flex Node agent systemd assets", + }, nil + } + if err != nil { + return agentbinary.ServicePlan{}, fmt.Errorf("read daemon asset %s: %w", asset.path, err) + } + } + return agentbinary.ServicePlan{Description: "AKS Flex Node agent systemd assets are current"}, nil +} + +func (s *flexDaemonActivationService) Prepare(_ context.Context, currentBinaryPath string) error { + return writeAgentServiceAssets(s.paths, s.systemdDir, s.recoveryScript, currentBinaryPath) +} + +func (s *flexDaemonActivationService) Reload(ctx context.Context) error { + if err := utilexec.ReloadSystemd(ctx, s.log); err != nil { + return fmt.Errorf("systemctl daemon-reload: %w", err) + } + return nil +} + +func (s *flexDaemonActivationService) Restart(ctx context.Context) error { + if err := utilexec.RunCmd(ctx, s.log, utilexec.Systemctl(), "restart", ServiceUnitName); err != nil { + return fmt.Errorf("systemctl restart %s: %w", ServiceUnitName, err) + } + return nil +} + +// WaitHealthy uses Flex's service and persisted active nspawn side. The nspawn +// exec-credential binary is synchronized only after systemd is stably running +// the expected host binary; shared rollback calls this again with last-good. +func (s *flexDaemonActivationService) WaitHealthy(ctx context.Context, expectedBinaryPath string) error { + healthCtx, cancel := context.WithTimeout(ctx, hostAgentHealthTimeout) + defer cancel() + expected, err := filepath.EvalSymlinks(expectedBinaryPath) + if err != nil { + return fmt.Errorf("resolve expected daemon binary: %w", err) + } + var healthySince time.Time + ticker := time.NewTicker(hostAgentHealthPoll) + defer ticker.Stop() + for { + healthy, checkErr := s.isExpectedDaemonActive(healthCtx, expected) + if checkErr == nil && healthy { + if healthySince.IsZero() { + healthySince = time.Now() + } else if time.Since(healthySince) >= hostAgentStableDuration { + state, stateErr := s.state.Load(healthCtx) + if stateErr != nil { + return fmt.Errorf("load active nspawn state: %w", stateErr) + } + if state == nil || !validNspawnMachine(state.ActiveMachine) { + return fmt.Errorf("no valid active nspawn machine for agent activation") + } + if syncErr := synchronizeNspawnAgentBinary(expected, state.ActiveMachine); syncErr != nil { + return fmt.Errorf("synchronize active nspawn agent binary: %w", syncErr) + } + return nil + } + } else { + healthySince = time.Time{} + } + select { + case <-healthCtx.Done(): + if checkErr != nil { + return fmt.Errorf("daemon did not become healthy: %w", checkErr) + } + return fmt.Errorf("daemon did not execute expected binary %s: %w", expected, healthCtx.Err()) + case <-ticker.C: + } + } +} + +func (s *flexDaemonActivationService) isExpectedDaemonActive(ctx context.Context, expected string) (bool, error) { + output, err := utilexec.OutputCmd(ctx, s.log, "systemctl", "show", "--property", "MainPID", "--value", ServiceUnitName) + if err != nil { + return false, err + } + pid, err := strconv.Atoi(strings.TrimSpace(output)) + if err != nil || pid <= 0 { + return false, fmt.Errorf("invalid daemon MainPID %q", output) + } + running, err := filepath.EvalSymlinks(fmt.Sprintf("/proc/%d/exe", pid)) + if err != nil { + return false, err + } + return running == expected, nil +} + +var _ agentbinary.DaemonService = (*flexDaemonActivationService)(nil) diff --git a/pkg/daemon/host_agent_activation_test.go b/pkg/daemon/host_agent_activation_test.go new file mode 100644 index 00000000..10ca5a67 --- /dev/null +++ b/pkg/daemon/host_agent_activation_test.go @@ -0,0 +1,56 @@ +package daemon + +import ( + "errors" + "log/slog" + "os" + "path/filepath" + "testing" +) + +func TestFlexDaemonActivationPreflightUsesFlexAssetsWithoutMutation(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + systemdDir := filepath.Join(t.TempDir(), "systemd") + recoveryScript := filepath.Join(t.TempDir(), "recovery.sh") + service := &flexDaemonActivationService{ + log: slog.Default(), + paths: paths, + systemdDir: systemdDir, + recoveryScript: recoveryScript, + } + plan, err := service.Preflight(t.Context(), paths.CurrentPath) + if err != nil { + t.Fatalf("Preflight: %v", err) + } + if !plan.UpdateRequired { + t.Fatal("UpdateRequired = false for missing Flex service assets") + } + for _, path := range []string{systemdDir, recoveryScript, paths.CurrentPath} { + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("preflight mutated %s: %v", path, err) + } + } +} + +func TestFlexDaemonActivationPreflightRejectsMachineOperationSignal(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.SignalPath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.SignalPath, []byte("{}"), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + service := &flexDaemonActivationService{ + log: slog.Default(), + paths: paths, + systemdDir: t.TempDir(), + recoveryScript: filepath.Join(t.TempDir(), "recovery.sh"), + } + if _, err := service.Preflight(t.Context(), paths.CurrentPath); err == nil { + t.Fatal("Preflight accepted a pending MachineOperation signal") + } +} diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index d9b743da..8c5282d4 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -77,29 +77,44 @@ func ensureAgentUpgradeServiceAssetsAt( if err := ensureAgentUpgradeLayout(ctx, log, binaryPaths); err != nil { return fmt.Errorf("initialize agent binary layout: %w", err) } - recoveryServiceContent := bytes.ReplaceAll( - recoveryServiceUnitContent, - []byte(recoveryScriptPath), - []byte(recoveryScript), - ) - assets := []struct { - path string - content []byte - mode os.FileMode - }{ - {path: filepath.Join(systemdDir, ServiceUnitName), content: serviceUnitContent, mode: 0o644}, + if err := writeAgentServiceAssets(binaryPaths, systemdDir, recoveryScript, binaryPaths.CurrentPath); err != nil { + return err + } + if err := reload(ctx, log); err != nil { + return fmt.Errorf("systemctl daemon-reload: %w", err) + } + return nil +} + +type agentServiceAsset struct { + path string + content []byte + mode os.FileMode +} + +func desiredAgentServiceAssets(binaryPaths agentUpgradePaths, systemdDir, recoveryScript, currentBinaryPath string) []agentServiceAsset { + serviceContent := bytes.ReplaceAll(serviceUnitContent, []byte(defaultAgentUpgradePaths().BinaryPath), []byte(currentBinaryPath)) + recoveryServiceContent := bytes.ReplaceAll(recoveryServiceUnitContent, []byte(recoveryScriptPath), []byte(recoveryScript)) + recoveryContent := recoveryScriptContent + for oldPath, newPath := range map[string]string{ + defaultAgentUpgradePaths().LastGoodPath: binaryPaths.LastGoodPath, + defaultAgentUpgradePaths().SignalPath: binaryPaths.SignalPath, + } { + recoveryContent = bytes.ReplaceAll(recoveryContent, []byte(oldPath), []byte(newPath)) + } + return []agentServiceAsset{ + {path: filepath.Join(systemdDir, ServiceUnitName), content: serviceContent, mode: 0o644}, {path: filepath.Join(systemdDir, recoveryServiceUnitName), content: recoveryServiceContent, mode: 0o644}, - {path: recoveryScript, content: recoveryScriptContent, mode: 0o750}, + {path: recoveryScript, content: recoveryContent, mode: 0o750}, } - for _, asset := range assets { +} + +func writeAgentServiceAssets(binaryPaths agentUpgradePaths, systemdDir, recoveryScript, currentBinaryPath string) error { + for _, asset := range desiredAgentServiceAssets(binaryPaths, systemdDir, recoveryScript, currentBinaryPath) { if err := utilio.WriteFile(asset.path, asset.content, asset.mode); err != nil { return fmt.Errorf("write %s: %w", asset.path, err) } } - - if err := reload(ctx, log); err != nil { - return fmt.Errorf("systemctl daemon-reload: %w", err) - } return nil } diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go index 043e0d27..cd888658 100644 --- a/pkg/daemon/lifecycle_test.go +++ b/pkg/daemon/lifecycle_test.go @@ -55,6 +55,9 @@ func TestEnsureAgentUpgradeServiceAssetsMigratesExistingInstallation(t *testing. if !strings.Contains(string(unit), "OnFailure="+recoveryServiceUnitName) { t.Fatalf("updated unit does not include recovery: %s", unit) } + if !strings.Contains(string(unit), "ExecStart="+paths.CurrentPath+" agent") { + t.Fatalf("updated unit does not execute the managed current link: %s", unit) + } recoveryService, err := os.ReadFile(filepath.Join(systemdDir, recoveryServiceUnitName)) if err != nil { t.Fatalf("recovery service was not installed: %v", err) diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index dc32a718..ade25ebe 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -13,6 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" machinav1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/daemon" ) @@ -151,6 +153,18 @@ func (h *machineOperationHandlers) reconcileAgentUpgrade( if err != nil { return h.finishFailedMachineOperation(ctx, store, op, "InvalidParameters", err.Error()) } + activationLock, err := h.agentUpgrade.Acquire() + if errors.Is(err, agentbinary.ErrActivationInProgress) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + if err != nil { + return ctrl.Result{}, fmt.Errorf("acquire agent activation lock: %w", err) + } + defer func() { + if closeErr := activationLock.Close(); closeErr != nil { + h.log.Warn("failed to release agent activation lock", "error", closeErr) + } + }() if err := store.MarkInProgress(ctx, op, "staging upgraded AKS Flex Node agent binary"); err != nil { return ctrl.Result{}, fmt.Errorf("mark AgentUpgrade MachineOperation in progress: %w", err) } diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index d2a4f6da..671ce495 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -3,6 +3,7 @@ package daemon import ( "context" "errors" + "io" "log/slog" "strings" "testing" @@ -12,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" machinav1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/daemon" ) @@ -356,6 +358,28 @@ func TestMachineOperationHandlersAgentUpgradeDuplicateReconcileIsNoop(t *testing } } +func TestMachineOperationHandlersAgentUpgradeRequeuesWhenDirectActivationHoldsLock(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{acquireErr: agentbinary.ErrActivationInProgress} + store := &fakeMachineOperationStore{} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + op := daemon.MachineOperation{Name: "upgrade-1", Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + agentUpgradeSHA256Parameter: strings.Repeat("a", 64), + }} + result, err := target.reconcileAgentUpgrade(t.Context(), store, op) + if err != nil { + t.Fatalf("reconcileAgentUpgrade: %v", err) + } + if result.RequeueAfter <= 0 { + t.Fatalf("RequeueAfter = %v, want positive duration", result.RequeueAfter) + } + if store.inProgress || upgrader.pending || upgrader.staged || upgrader.restarted { + t.Fatal("lock contention mutated the operation or staged an upgrade") + } +} + func TestMachineOperationHandlersAgentUpgradeStageFailureRollsBack(t *testing.T) { t.Parallel() @@ -473,12 +497,20 @@ type fakeAgentUpgradeExecutor struct { aborted bool restarted bool failure string + acquireErr error pendingErr error stageErr error abortErr error restartErr error } +func (f *fakeAgentUpgradeExecutor) Acquire() (io.Closer, error) { + if f.acquireErr != nil { + return nil, f.acquireErr + } + return io.NopCloser(strings.NewReader("")), nil +} + func (f *fakeAgentUpgradeExecutor) RecordPending(context.Context, string) error { f.pending = true return f.pendingErr From 69d9ed1a5f162a9d80acf2b5fdd0278436ba7480 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 18:09:13 +0000 Subject: [PATCH 23/45] test: validate direct host agent activation --- hack/e2e/README.md | 5 ++-- hack/e2e/lib/agent-upgrade.sh | 55 +++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/hack/e2e/README.md b/hack/e2e/README.md index b1238c1d..a6b84950 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -78,7 +78,7 @@ The default `all` command runs: | `validate` | Verify joined nodes, node-problem-detector status, and run smoke tests. | | `validate-absent` | Verify Flex Node objects are absent after unjoin. | | `smoke` | Run smoke workloads only. | -| `agent-upgrade` | Validate managed agent upgrade, forced rollback, retry, and nspawn synchronization. | +| `agent-upgrade` | Validate managed agent upgrade, forced rollback, retry, direct host activation, and nspawn synchronization. | | `upgrade-drift` | Validate controller-machine-driven repave to the alternate nspawn side. | | `logs` | Collect logs from VMs. | | `cleanup` | Collect logs and delete Azure resources. | @@ -156,7 +156,8 @@ The `agent-upgrade` command uses the bootstrap-token VM to exercise the complete 4. Restart kubelet to exercise the synchronized nspawn exec-credential binary and require the Node to remain Ready. 5. Upgrade to a candidate that passes `version` but fails daemon startup, then verify automatic rollback and a failed operation. 6. Confirm status does not expose the sensitive URL query and retry successfully into the inactive slot. -7. Run a workload before the subsequent repave test. +7. Stage a distinct candidate and validate direct host activation preflight, inactive-slot switch, service health, shared layout, and active-nspawn synchronization without creating a `MachineOperation` signal. +8. Restart kubelet through the directly activated nspawn credential binary, require Lease renewal and Node readiness, then run a workload before the subsequent repave test. Run it against an already joined environment: diff --git a/hack/e2e/lib/agent-upgrade.sh b/hack/e2e/lib/agent-upgrade.sh index a5416763..351927c7 100644 --- a/hack/e2e/lib/agent-upgrade.sh +++ b/hack/e2e/lib/agent-upgrade.sh @@ -177,8 +177,57 @@ REMOTE return 1 } +_agent_upgrade_direct_activation() { + local vm_name="$1" vm_ip="$2" before_snapshot before_slot before_digest after_snapshot after_slot after_digest + before_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" + IFS='|' read -r before_slot _ before_digest _ <<<"${before_snapshot}" + + remote_exec "${vm_ip}" 'bash -s' <<'REMOTE' +set -euo pipefail +work=/opt/aks-flex-node-e2e-upgrade +candidate="${work}/aks-flex-node-direct-candidate" +current_link=/usr/local/lib/aks-flex-node/aks-flex-node-current +last_good_link=/usr/local/lib/aks-flex-node/aks-flex-node-last-good +service=/etc/systemd/system/aks-flex-node-agent.service + +sudo cp "${work}/aks-flex-node-linux-amd64" "${candidate}" +printf '\nAKS-FLEX-DIRECT-ACTIVATION-E2E\n' | sudo tee -a "${candidate}" >/dev/null +sudo chmod 0755 "${candidate}" +current_before="$(sudo readlink -f "${current_link}")" +last_good_before="$(sudo readlink -f "${last_good_link}")" +unit_before="$(sudo sha256sum "${service}" | awk '{print $1}')" + +sudo "${candidate}" agent-upgrade --preflight | tee /tmp/direct-agent-upgrade-preflight.log +[[ "$(sudo readlink -f "${current_link}")" == "${current_before}" ]] +[[ "$(sudo readlink -f "${last_good_link}")" == "${last_good_before}" ]] +[[ "$(sudo sha256sum "${service}" | awk '{print $1}')" == "${unit_before}" ]] +sudo "${candidate}" agent-upgrade | tee /tmp/direct-agent-upgrade.log +current_after="$(sudo readlink -f "${current_link}")" +[[ "${current_after}" != "${current_before}" ]] +[[ "$(sudo readlink -f "${last_good_link}")" == "${current_before}" ]] +[[ "$(sudo readlink -f /usr/local/bin/aks-flex-node)" == "${current_after}" ]] +[[ "$(sudo sha256sum "${candidate}" | awk '{print $1}')" == "$(sudo sha256sum "${current_after}" | awk '{print $1}')" ]] +sudo grep -Fq "ExecStart=${current_link} agent" "${service}" +sudo systemctl is-active --quiet aks-flex-node-agent.service +pid="$(sudo systemctl show --property MainPID --value aks-flex-node-agent.service)" +[[ "$(sudo readlink -f "/proc/${pid}/exe")" == "${current_after}" ]] +[[ ! -e /etc/aks-flex-node/agent-upgrade-signal.json ]] +REMOTE + + after_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" + IFS='|' read -r after_slot _ after_digest _ <<<"${after_snapshot}" + if [[ -z "${after_slot}" || "${after_slot}" == "${before_slot}" || "${after_digest}" == "${before_digest}" ]]; then + log_error "Direct activation did not install a distinct inactive-slot candidate: before=${before_snapshot} after=${after_snapshot}" + return 1 + fi + _agent_upgrade_assert_synchronized "${vm_ip}" + _agent_upgrade_validate_kubelet_auth "${vm_name}" "${vm_ip}" + validate_node_joined "${vm_name}" + log_success "Direct host activation preflight, switch, service health, and nspawn synchronization passed" +} + agent_upgrade_e2e() { - log_section "Managed AgentUpgrade E2E" + log_section "Managed and Direct AgentUpgrade E2E" local vm_name vm_ip suffix success_digest failure_digest before before_slot success_snapshot success_slot success_binary_digest rollback_snapshot rollback_binary_digest retry_snapshot retry_binary_digest vm_name="$(state_get token_vm_name)" vm_ip="$(state_get token_vm_ip)" @@ -237,7 +286,9 @@ agent_upgrade_e2e() { fi _agent_upgrade_assert_synchronized "${vm_ip}" validate_node_joined "${vm_name}" + + _agent_upgrade_direct_activation "${vm_name}" "${vm_ip}" smoke_test "${vm_name}" "agent-upgrade" - log_success "Managed AgentUpgrade success, rollback, and retry E2E passed" + log_success "Managed AgentUpgrade success/rollback/retry and direct host activation E2E passed" } From 7dd2660fa5a75404471f0877ab4f966bc8b06057 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 18:39:41 +0000 Subject: [PATCH 24/45] chore: update host activation dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 797efc58..bdd5c6e0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260810172605-4ad184157a14 + github.com/Azure/unbounded v0.2.3-0.20260810183719-7479dd412756 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index dd9f8089..8a14c783 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260810172605-4ad184157a14 h1:APWKYrmdrkCq3uZAnRvNFuFvOY+Z8IKEIEjnKxKEIro= -github.com/Azure/unbounded v0.2.3-0.20260810172605-4ad184157a14/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= +github.com/Azure/unbounded v0.2.3-0.20260810183719-7479dd412756 h1:GfC3UaRaBK2Fb4Q90q74iLlDFoKDQ+zFE8e8fgLEpp4= +github.com/Azure/unbounded v0.2.3-0.20260810183719-7479dd412756/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From 65530492781631d79e0f8311b929ffcf070c8bc3 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 21:50:00 +0000 Subject: [PATCH 25/45] chore: pin merged host activation API --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bdd5c6e0..c325edbb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.2.3-0.20260810183719-7479dd412756 + github.com/Azure/unbounded v0.2.3-0.20260810210923-b05575d69100 github.com/go-logr/logr v1.4.4 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 8a14c783..43b37a60 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.2.3-0.20260810183719-7479dd412756 h1:GfC3UaRaBK2Fb4Q90q74iLlDFoKDQ+zFE8e8fgLEpp4= -github.com/Azure/unbounded v0.2.3-0.20260810183719-7479dd412756/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= +github.com/Azure/unbounded v0.2.3-0.20260810210923-b05575d69100 h1:IJhwIYAYEsJ9cJSoNdieB/Y95acWQf1tEaOkp395EJQ= +github.com/Azure/unbounded v0.2.3-0.20260810210923-b05575d69100/go.mod h1:G+yWlIQB/KwOk+O7HPvcRLBW88qZUgSWt51qytBTgFE= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= From dc9a8a3ffe4350e0e4781863d4a92eff0c1a61bf Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 21:56:28 +0000 Subject: [PATCH 26/45] refactor: align upgrade commands with merged API --- cmd/aks-flex-node/main.go | 4 +--- docs/usages/operations.md | 4 ++-- hack/e2e/README.md | 2 +- hack/e2e/lib/agent-upgrade.sh | 11 ++++++--- pkg/cmd/daemon/daemon.go | 9 +++++++ pkg/cmd/daemon/daemon_test.go | 31 +++++++++++++++++++++++++ pkg/daemon/agent_upgrade.go | 3 --- pkg/daemon/agent_upgrade_binary.go | 8 ++++--- pkg/daemon/agent_upgrade_binary_test.go | 7 ++++++ pkg/daemon/agent_upgrade_test.go | 3 +-- 10 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 pkg/cmd/daemon/daemon_test.go diff --git a/cmd/aks-flex-node/main.go b/cmd/aks-flex-node/main.go index 2dc07885..8b00a6aa 100644 --- a/cmd/aks-flex-node/main.go +++ b/cmd/aks-flex-node/main.go @@ -29,9 +29,7 @@ func main() { rootCmd.AddCommand(start.NewCommand()) rootCmd.AddCommand(bootstrapdata.NewCommand()) rootCmd.AddCommand(preflight.NewCommand()) - rootCmd.AddCommand(daemon.NewCommand()) - rootCmd.AddCommand(daemon.NewHostAgentUpgradeCommand()) - rootCmd.AddCommand(daemon.NewAgentUpgradeRecoveryCommand()) + rootCmd.AddCommand(daemon.NewCommands()...) rootCmd.AddCommand(reset.NewCommand()) rootCmd.AddCommand(version.NewCommand()) rootCmd.AddCommand(token.Command) diff --git a/docs/usages/operations.md b/docs/usages/operations.md index 84b51bf2..3884d4d8 100644 --- a/docs/usages/operations.md +++ b/docs/usages/operations.md @@ -49,7 +49,7 @@ journalctl -u aks-flex-node-agent -f ## Managed Agent Upgrade -When the Unbounded `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTPS release archive and the SHA-256 of the compressed archive: +When the Unbounded `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTPS release archive and, when available, the SHA-256 of the compressed archive: ```yaml apiVersion: unbounded-cloud.io/v1alpha3 @@ -64,7 +64,7 @@ spec: sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef ``` -The archive must contain exactly the architecture-specific release member used by AKS Flex Node (`aks-flex-node-linux-amd64` or `aks-flex-node-linux-arm64`). The daemon verifies the archive digest and candidate `version` command before switching its blue/green binary links. It also atomically updates the binary in the active nspawn rootfs so kubelet exec authentication uses the same version. +The archive must contain exactly the architecture-specific release member used by AKS Flex Node (`aks-flex-node-linux-amd64` or `aks-flex-node-linux-arm64`). The `sha256` parameter is optional; when supplied, the daemon verifies the compressed archive digest. Omit it only when the archive source and HTTPS transport are trusted. The daemon always verifies the candidate `version` command before switching its blue/green binary links. It also atomically updates the binary in the active nspawn rootfs so kubelet exec authentication uses the same version. The restarted daemon marks the operation `Complete`. If the candidate cannot remain running, systemd restores the last-known-good host and nspawn binaries and marks the operation `Failed`. URL query strings, which may contain SAS credentials, are omitted from logs and operation status. diff --git a/hack/e2e/README.md b/hack/e2e/README.md index a6b84950..adaa2831 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -155,7 +155,7 @@ The `agent-upgrade` command uses the bootstrap-token VM to exercise the complete 3. Verify successful daemon restart, operation completion, binary replacement, and host/nspawn binary equality. 4. Restart kubelet to exercise the synchronized nspawn exec-credential binary and require the Node to remain Ready. 5. Upgrade to a candidate that passes `version` but fails daemon startup, then verify automatic rollback and a failed operation. -6. Confirm status does not expose the sensitive URL query and retry successfully into the inactive slot. +6. Confirm status does not expose the sensitive URL query and retry successfully into the inactive slot without the optional archive digest. 7. Stage a distinct candidate and validate direct host activation preflight, inactive-slot switch, service health, shared layout, and active-nspawn synchronization without creating a `MachineOperation` signal. 8. Restart kubelet through the directly activated nspawn credential binary, require Lease renewal and Node readiness, then run a workload before the subsequent repave test. diff --git a/hack/e2e/lib/agent-upgrade.sh b/hack/e2e/lib/agent-upgrade.sh index 351927c7..597fa0eb 100644 --- a/hack/e2e/lib/agent-upgrade.sh +++ b/hack/e2e/lib/agent-upgrade.sh @@ -92,7 +92,10 @@ _agent_upgrade_digest() { } _agent_upgrade_apply() { - local operation="$1" vm_name="$2" archive="$3" digest="$4" token="$5" + local operation="$1" vm_name="$2" archive="$3" digest="$4" token="$5" digest_parameter="" + if [[ -n "${digest}" ]]; then + digest_parameter=" sha256: ${digest}" + fi cat < Date: Mon, 10 Aug 2026 22:27:50 +0000 Subject: [PATCH 27/45] feat: support HTTP agent upgrade archives --- docs/usages/operations.md | 4 ++-- hack/e2e/README.md | 2 +- hack/e2e/lib/agent-upgrade.sh | 30 +++++-------------------- pkg/daemon/agent_upgrade_binary.go | 17 +++----------- pkg/daemon/agent_upgrade_binary_test.go | 15 +++++-------- pkg/daemon/agent_upgrade_test.go | 1 - 6 files changed, 16 insertions(+), 53 deletions(-) diff --git a/docs/usages/operations.md b/docs/usages/operations.md index 3884d4d8..3a7649fe 100644 --- a/docs/usages/operations.md +++ b/docs/usages/operations.md @@ -49,7 +49,7 @@ journalctl -u aks-flex-node-agent -f ## Managed Agent Upgrade -When the Unbounded `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTPS release archive and, when available, the SHA-256 of the compressed archive: +When the Unbounded `MachineOperation` API is installed, submit an `AgentUpgrade` with an HTTP or HTTPS release archive and, when available, the SHA-256 of the compressed archive: ```yaml apiVersion: unbounded-cloud.io/v1alpha3 @@ -64,7 +64,7 @@ spec: sha256: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef ``` -The archive must contain exactly the architecture-specific release member used by AKS Flex Node (`aks-flex-node-linux-amd64` or `aks-flex-node-linux-arm64`). The `sha256` parameter is optional; when supplied, the daemon verifies the compressed archive digest. Omit it only when the archive source and HTTPS transport are trusted. The daemon always verifies the candidate `version` command before switching its blue/green binary links. It also atomically updates the binary in the active nspawn rootfs so kubelet exec authentication uses the same version. +The archive must contain exactly the architecture-specific release member used by AKS Flex Node (`aks-flex-node-linux-amd64` or `aks-flex-node-linux-arm64`). The `sha256` parameter is optional; when supplied, the daemon verifies the compressed archive digest. Prefer HTTPS and a digest for production downloads. Plain HTTP is intended for explicitly trusted networks such as a VM-local loopback server; omit the digest only when both the archive source and transport path are trusted. The daemon always verifies the candidate `version` command before switching its blue/green binary links. It also atomically updates the binary in the active nspawn rootfs so kubelet exec authentication uses the same version. The restarted daemon marks the operation `Complete`. If the candidate cannot remain running, systemd restores the last-known-good host and nspawn binaries and marks the operation `Failed`. URL query strings, which may contain SAS credentials, are omitted from logs and operation status. diff --git a/hack/e2e/README.md b/hack/e2e/README.md index adaa2831..34d88bad 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -150,7 +150,7 @@ Each join path uploads the locally built binary, renders a config file, installs The `agent-upgrade` command uses the bootstrap-token VM to exercise the complete managed binary lifecycle: -1. Serve architecture-specific release archives over trusted loopback HTTPS. +1. Serve architecture-specific release archives over VM-local loopback HTTP to validate HTTP transport support. 2. Submit an `AgentUpgrade` with an archive SHA-256 and a query credential. 3. Verify successful daemon restart, operation completion, binary replacement, and host/nspawn binary equality. 4. Restart kubelet to exercise the synchronized nspawn exec-credential binary and require the Node to remain Ready. diff --git a/hack/e2e/lib/agent-upgrade.sh b/hack/e2e/lib/agent-upgrade.sh index 597fa0eb..cbdaaeee 100644 --- a/hack/e2e/lib/agent-upgrade.sh +++ b/hack/e2e/lib/agent-upgrade.sh @@ -51,32 +51,12 @@ fi rm -rf "${check_dir}" sudo install -m 0755 /tmp/aks-flex-node-e2e-upgrade-binary "${work}/aks-flex-node-linux-amd64" -sudo openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ - -subj '/CN=127.0.0.1' \ - -addext 'subjectAltName=IP:127.0.0.1' \ - -keyout "${work}/server.key" -out "${work}/server.crt" >/dev/null 2>&1 -sudo cp "${work}/server.crt" /usr/local/share/ca-certificates/aks-flex-node-e2e-upgrade.crt -sudo update-ca-certificates >/dev/null -# Reload Go's system root pool in the long-running daemon after adding the -# short-lived test CA. -sudo systemctl restart aks-flex-node-agent.service -cat >/tmp/aks-flex-node-e2e-upgrade-server.py <<'PY' -import http.server -import ssl - -server = http.server.ThreadingHTTPServer(("127.0.0.1", 18443), http.server.SimpleHTTPRequestHandler) -context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) -context.load_cert_chain("/opt/aks-flex-node-e2e-upgrade/server.crt", "/opt/aks-flex-node-e2e-upgrade/server.key") -server.socket = context.wrap_socket(server.socket, server_side=True) -server.serve_forever() -PY -sudo install -m 0755 /tmp/aks-flex-node-e2e-upgrade-server.py "${work}/server.py" sudo systemctl stop aks-flex-node-e2e-upgrade-server.service 2>/dev/null || true sudo systemd-run --unit=aks-flex-node-e2e-upgrade-server.service \ --property=WorkingDirectory="${work}" \ - /usr/bin/python3 "${work}/server.py" >/dev/null + /usr/bin/python3 -m http.server 18080 --bind 127.0.0.1 >/dev/null for _ in $(seq 1 30); do - if curl --silent --fail https://127.0.0.1:18443/success.tar.gz >/dev/null; then + if curl --silent --fail http://127.0.0.1:18080/success.tar.gz >/dev/null; then exit 0 fi sleep 1 @@ -105,7 +85,7 @@ spec: machineRef: ${vm_name} operationKind: AgentUpgrade parameters: - downloadURL: https://127.0.0.1:18443/${archive}?sig=${token} + downloadURL: http://127.0.0.1:18080/${archive}?sig=${token} ${digest_parameter} ttlSecondsAfterFinished: 3600 EOF @@ -279,8 +259,8 @@ agent_upgrade_e2e() { validate_node_joined "${vm_name}" local retry_op="agent-upgrade-retry-${suffix}" - # The digest is optional when the trusted archive source and HTTPS transport - # provide the integrity boundary. + # The digest is optional when the VM-local source and loopback transport + # provide the trust boundary. _agent_upgrade_apply "${retry_op}" "${vm_name}" success.tar.gz "" "retry-${suffix}" _agent_upgrade_wait_phase "${retry_op}" Complete retry_snapshot="$(_agent_upgrade_snapshot "${vm_ip}")" diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 0cf89bec..c2b08d78 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -182,8 +182,8 @@ func expectedAgentArchiveMember() (string, error) { func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.InstallOptions, error) { parsedURL, err := url.ParseRequestURI(strings.TrimSpace(rawURL)) - if err != nil || parsedURL.Scheme != "https" || parsedURL.Host == "" || parsedURL.User != nil || parsedURL.Fragment != "" { - return agentbinary.InstallOptions{}, fmt.Errorf("download URL must use HTTPS, include a host, omit user information, and omit fragments") + if err != nil || parsedURL.Scheme != "http" && parsedURL.Scheme != "https" || parsedURL.Host == "" || parsedURL.User != nil || parsedURL.Fragment != "" { + return agentbinary.InstallOptions{}, fmt.Errorf("download URL must use HTTP or HTTPS, include a host, omit user information, and omit fragments") } digest := strings.TrimPrefix(strings.TrimSpace(expectedDigest), "sha256:") if digest != "" { @@ -204,18 +204,7 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta MaxArchiveBytes: agentUpgradeMaxArchiveBytes, MaxExtractedBytes: agentUpgradeMaxBinaryBytes, ExactMember: true, - HTTPClient: &http.Client{ - Timeout: 10 * time.Minute, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - if req.URL.Scheme != "https" { - return fmt.Errorf("redirect to non-HTTPS URL is not allowed") - } - if len(via) >= 10 { - return fmt.Errorf("stopped after 10 redirects") - } - return nil - }, - }, + HTTPClient: &http.Client{Timeout: 10 * time.Minute}, }, nil } diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go index 8bd8592e..9da9fb24 100644 --- a/pkg/daemon/agent_upgrade_binary_test.go +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -2,7 +2,6 @@ package daemon import ( "log/slog" - "net/http" "os" "path/filepath" "runtime" @@ -63,12 +62,8 @@ func TestSecureAgentInstallOptions(t *testing.T) { if withoutDigest.ExpectedSHA256 != "" { t.Fatalf("ExpectedSHA256 = %q, want empty", withoutDigest.ExpectedSHA256) } - redirect, err := http.NewRequest(http.MethodGet, "http://example.com/agent.tar.gz", http.NoBody) - if err != nil { - t.Fatalf("NewRequest: %v", err) - } - if err := opts.HTTPClient.CheckRedirect(redirect, nil); err == nil { - t.Fatal("HTTP redirect was accepted") + if _, err := secureAgentInstallOptions("http://127.0.0.1/agent.tar.gz", digest); err != nil { + t.Fatalf("secureAgentInstallOptions with HTTP: %v", err) } } @@ -79,8 +74,8 @@ func TestSecureAgentInstallOptionsRejectsInvalidInputs(t *testing.T) { url string digest string }{ - "HTTP": { - url: "http://example.com/agent.tar.gz", + "unsupported scheme": { + url: "ftp://example.com/agent.tar.gz", digest: strings.Repeat("a", 64), }, "invalid digest": { @@ -115,7 +110,7 @@ func TestInstallAndSwitchAgentBinaryRejectsInvalidInputsWithoutSwitching(t *test err := installAndSwitchAgentBinary( t.Context(), slog.Default(), - "http://example.com/agent.tar.gz", + "ftp://example.com/agent.tar.gz", strings.Repeat("0", 64), paths, ) diff --git a/pkg/daemon/agent_upgrade_test.go b/pkg/daemon/agent_upgrade_test.go index d527090a..cec9d398 100644 --- a/pkg/daemon/agent_upgrade_test.go +++ b/pkg/daemon/agent_upgrade_test.go @@ -53,7 +53,6 @@ func TestParseAgentUpgradeRequest(t *testing.T) { agentUpgradeDownloadURLParameter: "http://example.com/agent.tar.gz", agentUpgradeSHA256Parameter: strings.Repeat("a", 64), }, - wantErr: "HTTPS", }, "optional digest omitted": { parameters: map[string]string{agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz"}, From 947bfb9f0e506c0dc6f2b7715c3fd55ca1b120ac Mon Sep 17 00:00:00 2001 From: Baichao He Date: Mon, 10 Aug 2026 22:58:49 +0000 Subject: [PATCH 28/45] fix: harden upgrade recovery and reinstall --- pkg/daemon/agent_upgrade_binary.go | 20 ++++++-- pkg/daemon/agent_upgrade_binary_test.go | 23 +++++++++ pkg/daemon/machineoperation_reconciler.go | 15 ++++-- .../machineoperation_reconciler_test.go | 22 +++++++++ scripts/install.sh | 49 +++++++++++++++++-- 5 files changed, 119 insertions(+), 10 deletions(-) diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index c2b08d78..80f66676 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -47,8 +47,8 @@ func (p agentUpgradePaths) layout() agentbinary.Layout { } } -func (p agentUpgradePaths) sharedPaths() goalstates.AgentUpgradePaths { - return goalstates.AgentUpgradePaths{ +func (p agentUpgradePaths) sharedPaths() (goalstates.AgentUpgradePaths, error) { + paths := goalstates.AgentUpgradePaths{ BinaryPath: p.BinaryPath, BluePath: p.BluePath, GreenPath: p.GreenPath, @@ -56,6 +56,16 @@ func (p agentUpgradePaths) sharedPaths() goalstates.AgentUpgradePaths { LastGoodPath: p.LastGoodPath, SignalPath: p.SignalPath, } + target, err := filepath.EvalSymlinks(p.CurrentPath) + if err == nil { + paths.CurrentTargetPath = target + return paths, nil + } + if errors.Is(err, os.ErrNotExist) { + paths.CurrentTargetPath = p.BinaryPath + return paths, nil + } + return goalstates.AgentUpgradePaths{}, fmt.Errorf("resolve current agent binary: %w", err) } // ensureAgentUpgradeLayout adds Flex-specific ownership validation around the @@ -71,7 +81,11 @@ func ensureAgentUpgradeLayout(ctx context.Context, log *slog.Logger, paths agent if productionPaths && os.Geteuid() != 0 { return fmt.Errorf("agent binary layout must be initialized as root") } - if err := agentbinary.EnsureDaemonBinaryLinks(ctx, log, paths.sharedPaths()); err != nil { + sharedPaths, err := paths.sharedPaths() + if err != nil { + return err + } + if err := agentbinary.EnsureDaemonBinaryLinks(ctx, log, sharedPaths); err != nil { return err } if productionPaths { diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go index 9da9fb24..01440c76 100644 --- a/pkg/daemon/agent_upgrade_binary_test.go +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -38,6 +38,29 @@ func TestEnsureAgentUpgradeLayoutMigratesLegacyBinaryIdempotently(t *testing.T) } } +func TestEnsureAgentUpgradeLayoutRecoversInterruptedLastGoodInitialization(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.BluePath, []byte("active"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + + if err := ensureAgentUpgradeLayout(t.Context(), slog.Default(), paths); err != nil { + t.Fatalf("ensureAgentUpgradeLayout: %v", err) + } + + assertResolvedPath(t, paths.CurrentPath, paths.BluePath) + assertResolvedPath(t, paths.LastGoodPath, paths.BluePath) + assertResolvedPath(t, paths.BinaryPath, paths.BluePath) +} + func TestSecureAgentInstallOptions(t *testing.T) { t.Parallel() diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index ade25ebe..8539c8de 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -165,9 +165,9 @@ func (h *machineOperationHandlers) reconcileAgentUpgrade( h.log.Warn("failed to release agent activation lock", "error", closeErr) } }() - if err := store.MarkInProgress(ctx, op, "staging upgraded AKS Flex Node agent binary"); err != nil { - return ctrl.Result{}, fmt.Errorf("mark AgentUpgrade MachineOperation in progress: %w", err) - } + // Persist the recovery signal before InProgress. The shared reconciler does + // not enqueue InProgress operations after a process crash, so the signal + // must exist before the status can become non-reconcilable. if err := h.agentUpgrade.RecordPending(ctx, op.Name); err != nil { if errors.Is(err, errAgentUpgradeAlreadyPending) { // An InProgress status event can already be queued before the delayed @@ -177,6 +177,15 @@ func (h *machineOperationHandlers) reconcileAgentUpgrade( } return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", err.Error()) } + if err := store.MarkInProgress(ctx, op, "staging upgraded AKS Flex Node agent binary"); err != nil { + cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) + abortErr := h.agentUpgrade.Abort(cleanupCtx) + cancel() + return ctrl.Result{}, errors.Join( + fmt.Errorf("mark AgentUpgrade MachineOperation in progress: %w", err), + wrapOptionalError("clear pending AgentUpgrade signal", abortErr), + ) + } if err := h.agentUpgrade.Stage(ctx, request); err != nil { if abortErr := h.agentUpgrade.Abort(ctx); abortErr != nil { return h.beginAgentUpgradeRecovery(ctx, op, err, abortErr) diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index 671ce495..6a70c1df 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -380,6 +380,28 @@ func TestMachineOperationHandlersAgentUpgradeRequeuesWhenDirectActivationHoldsLo } } +func TestMachineOperationHandlersAgentUpgradeMarkFailureClearsPendingSignal(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{} + store := &fakeMachineOperationStore{markErr: errors.New("status update failed")} + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + op := daemon.MachineOperation{Name: "upgrade-1", Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + }} + + _, err := target.reconcileAgentUpgrade(t.Context(), store, op) + if err == nil || !strings.Contains(err.Error(), "mark AgentUpgrade MachineOperation in progress") { + t.Fatalf("reconcileAgentUpgrade error = %v", err) + } + if !upgrader.pending || !upgrader.aborted { + t.Fatalf("pending = %v, aborted = %v; want durable signal followed by cleanup", upgrader.pending, upgrader.aborted) + } + if upgrader.staged || upgrader.restarted { + t.Fatal("status failure staged or restarted the agent") + } +} + func TestMachineOperationHandlersAgentUpgradeStageFailureRollsBack(t *testing.T) { t.Parallel() diff --git a/scripts/install.sh b/scripts/install.sh index 3566535b..9653fcb5 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -254,15 +254,56 @@ download_binary() { install_binary() { local binary_path="$1" + local managed_dir="/usr/local/lib/aks-flex-node" + local current_path="${managed_dir}/aks-flex-node-current" + local blue_path="${managed_dir}/aks-flex-node-blue" + local green_path="${managed_dir}/aks-flex-node-green" + local last_good_path="${managed_dir}/aks-flex-node-last-good" + local compatibility_path="${INSTALL_DIR}/aks-flex-node" + local candidate log_info "Installing binary to $INSTALL_DIR..." - # Install with explicit ownership and modes so restrictive remote umasks do - # not leave the command inaccessible to non-root operators. + # Always stage separately. Installing directly through BinaryPath after + # migration would dereference its symlink and overwrite an active or + # last-good slot in place. + candidate=$(mktemp /var/tmp/aks-flex-node-install-candidate.XXXXXX) + install -o root -g root -m 0755 "$binary_path" "$candidate" + + if [[ -e "$current_path" || -L "$current_path" ]]; then + if systemctl is-active --quiet aks-flex-node-agent.service 2>/dev/null; then + log_info "Activating candidate through the managed blue-green layout..." + if ! "$candidate" agent-upgrade; then + rm -f "$candidate" + log_error "Failed to activate the installed AKS Flex Node candidate" + return 1 + fi + rm -f "$candidate" + log_success "Binary activated through $current_path" + return 0 + fi + + # Reset/unjoin removes daemon state and service assets but may leave the + # binary slots. With no running daemon to preserve, safely reseed the + # managed layout instead of writing through compatibility symlinks. + log_info "Reseeding inactive managed binary layout..." + install -d -o root -g root -m 0750 "$managed_dir" + rm -f "$compatibility_path" "$current_path" "$last_good_path" "$blue_path" "$green_path" + install -o root -g root -m 0755 "$candidate" "$blue_path" + ln -s "$blue_path" "$current_path" + ln -s "$blue_path" "$last_good_path" + install -d -o root -g root -m 0755 "$INSTALL_DIR" + ln -s "$current_path" "$compatibility_path" + rm -f "$candidate" + log_success "Binary layout reseeded at $current_path" + return 0 + fi + install -d -o root -g root -m 0755 "$INSTALL_DIR" - install -o root -g root -m 0755 "$binary_path" "$INSTALL_DIR/aks-flex-node" + install -o root -g root -m 0755 "$candidate" "$compatibility_path" + rm -f "$candidate" - log_success "Binary installed to $INSTALL_DIR/aks-flex-node" + log_success "Binary installed to $compatibility_path" } warn_install_dir_not_in_path() { From 46b3278d2e35bee8a0daff211456f8e55af11cdf Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 01:07:00 +0000 Subject: [PATCH 29/45] refactor: encapsulate daemon subcommands --- pkg/cmd/daemon/agent_upgrade_recovery.go | 6 ++-- pkg/cmd/daemon/daemon.go | 8 ++--- pkg/cmd/daemon/host_agent_upgrade.go | 41 ++++++++++++++---------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/pkg/cmd/daemon/agent_upgrade_recovery.go b/pkg/cmd/daemon/agent_upgrade_recovery.go index 8c6389cc..40287004 100644 --- a/pkg/cmd/daemon/agent_upgrade_recovery.go +++ b/pkg/cmd/daemon/agent_upgrade_recovery.go @@ -8,9 +8,9 @@ import ( hostdaemon "github.com/Azure/AKSFlexNode/pkg/daemon" ) -// NewAgentUpgradeRecoveryCommand returns the internal command used by the -// systemd recovery unit. It remains callable only as a local root operation. -func NewAgentUpgradeRecoveryCommand() *cobra.Command { +// newAgentUpgradeRecoveryCommand remains callable only as a local root +// operation by the systemd recovery unit. +func newAgentUpgradeRecoveryCommand() *cobra.Command { var message string cmd := &cobra.Command{ Use: "recover-agent-upgrade", diff --git a/pkg/cmd/daemon/daemon.go b/pkg/cmd/daemon/daemon.go index d3032482..af9ae234 100644 --- a/pkg/cmd/daemon/daemon.go +++ b/pkg/cmd/daemon/daemon.go @@ -13,13 +13,13 @@ import ( // NewCommands returns all daemon runtime and internal lifecycle commands. func NewCommands() []*cobra.Command { return []*cobra.Command{ - NewCommand(), - NewHostAgentUpgradeCommand(), - NewAgentUpgradeRecoveryCommand(), + newCommand(), + newHostAgentUpgradeCommand(), + newAgentUpgradeRecoveryCommand(), } } -func NewCommand() *cobra.Command { +func newCommand() *cobra.Command { var configPath string cmd := &cobra.Command{ Use: "daemon", diff --git a/pkg/cmd/daemon/host_agent_upgrade.go b/pkg/cmd/daemon/host_agent_upgrade.go index fc8e47fe..ea497d58 100644 --- a/pkg/cmd/daemon/host_agent_upgrade.go +++ b/pkg/cmd/daemon/host_agent_upgrade.go @@ -1,7 +1,10 @@ package daemon import ( + "context" "fmt" + "io" + "log/slog" "os" "path/filepath" @@ -11,8 +14,7 @@ import ( "github.com/Azure/AKSFlexNode/pkg/logger" ) -// NewHostAgentUpgradeCommand returns the hidden host-driven activation command. -func NewHostAgentUpgradeCommand() *cobra.Command { +func newHostAgentUpgradeCommand() *cobra.Command { var preflight bool cmd := &cobra.Command{ Use: "agent-upgrade", @@ -29,23 +31,11 @@ func NewHostAgentUpgradeCommand() *cobra.Command { return fmt.Errorf("resolve absolute candidate executable path: %w", err) } log := logger.CreateLogger("info", "") + candidate = filepath.Clean(candidate) if preflight { - plan, err := hostdaemon.PreflightHostAgentActivation(cmd.Context(), log, filepath.Clean(candidate)) - if err != nil { - return err - } - if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Candidate: %s\nActive binary: %s\nInstall target: %s\n", plan.CandidatePath, plan.ActivePath, plan.TargetPath); err != nil { - return err - } - for _, action := range plan.Actions { - if _, err := fmt.Fprintf(cmd.OutOrStdout(), "- %s\n", action); err != nil { - return err - } - } - _, err = fmt.Fprintln(cmd.OutOrStdout(), "Preflight: no changes applied") - return err + return runHostAgentUpgradePreflight(cmd.Context(), cmd.OutOrStdout(), log, candidate) } - result, err := hostdaemon.ActivateHostAgent(cmd.Context(), log, filepath.Clean(candidate)) + result, err := hostdaemon.ActivateHostAgent(cmd.Context(), log, candidate) if err != nil { return err } @@ -56,3 +46,20 @@ func NewHostAgentUpgradeCommand() *cobra.Command { cmd.Flags().BoolVar(&preflight, "preflight", false, "Show and validate the host activation plan without applying it") return cmd } + +func runHostAgentUpgradePreflight(ctx context.Context, output io.Writer, log *slog.Logger, candidate string) error { + plan, err := hostdaemon.PreflightHostAgentActivation(ctx, log, candidate) + if err != nil { + return err + } + if _, err := fmt.Fprintf(output, "Candidate: %s\nActive binary: %s\nInstall target: %s\n", plan.CandidatePath, plan.ActivePath, plan.TargetPath); err != nil { + return err + } + for _, action := range plan.Actions { + if _, err := fmt.Fprintf(output, "- %s\n", action); err != nil { + return err + } + } + _, err = fmt.Fprintln(output, "Preflight: no changes applied") + return err +} From 6a66d855ae3205c9beec37f2f4293f08b9186a43 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 01:12:36 +0000 Subject: [PATCH 30/45] refactor: delegate managed installs to activation --- pkg/daemon/host_agent_activation.go | 37 +++++++++++++++--------- pkg/daemon/host_agent_activation_test.go | 12 ++++++++ scripts/install.sh | 33 ++++----------------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pkg/daemon/host_agent_activation.go b/pkg/daemon/host_agent_activation.go index 0fcdaef6..6661ade3 100644 --- a/pkg/daemon/host_agent_activation.go +++ b/pkg/daemon/host_agent_activation.go @@ -119,9 +119,10 @@ func (s *flexDaemonActivationService) Restart(ctx context.Context) error { return nil } -// WaitHealthy uses Flex's service and persisted active nspawn side. The nspawn -// exec-credential binary is synchronized only after systemd is stably running -// the expected host binary; shared rollback calls this again with last-good. +// WaitHealthy uses Flex's service and persisted active nspawn side. When a +// side exists, its exec-credential binary is synchronized only after systemd +// is stably running the expected host binary; reset hosts have no side to sync. +// Shared rollback calls this again with last-good. func (s *flexDaemonActivationService) WaitHealthy(ctx context.Context, expectedBinaryPath string) error { healthCtx, cancel := context.WithTimeout(ctx, hostAgentHealthTimeout) defer cancel() @@ -138,17 +139,7 @@ func (s *flexDaemonActivationService) WaitHealthy(ctx context.Context, expectedB if healthySince.IsZero() { healthySince = time.Now() } else if time.Since(healthySince) >= hostAgentStableDuration { - state, stateErr := s.state.Load(healthCtx) - if stateErr != nil { - return fmt.Errorf("load active nspawn state: %w", stateErr) - } - if state == nil || !validNspawnMachine(state.ActiveMachine) { - return fmt.Errorf("no valid active nspawn machine for agent activation") - } - if syncErr := synchronizeNspawnAgentBinary(expected, state.ActiveMachine); syncErr != nil { - return fmt.Errorf("synchronize active nspawn agent binary: %w", syncErr) - } - return nil + return s.synchronizeActiveNspawn(healthCtx, expected) } } else { healthySince = time.Time{} @@ -164,6 +155,24 @@ func (s *flexDaemonActivationService) WaitHealthy(ctx context.Context, expectedB } } +func (s *flexDaemonActivationService) synchronizeActiveNspawn(ctx context.Context, expected string) error { + state, err := s.state.Load(ctx) + if err != nil { + return fmt.Errorf("load active nspawn state: %w", err) + } + if state == nil { + s.log.Info("activated host agent without active nspawn synchronization") + return nil + } + if !validNspawnMachine(state.ActiveMachine) { + return fmt.Errorf("no valid active nspawn machine for agent activation") + } + if err := synchronizeNspawnAgentBinary(expected, state.ActiveMachine); err != nil { + return fmt.Errorf("synchronize active nspawn agent binary: %w", err) + } + return nil +} + func (s *flexDaemonActivationService) isExpectedDaemonActive(ctx context.Context, expected string) (bool, error) { output, err := utilexec.OutputCmd(ctx, s.log, "systemctl", "show", "--property", "MainPID", "--value", ServiceUnitName) if err != nil { diff --git a/pkg/daemon/host_agent_activation_test.go b/pkg/daemon/host_agent_activation_test.go index 10ca5a67..3f306d7a 100644 --- a/pkg/daemon/host_agent_activation_test.go +++ b/pkg/daemon/host_agent_activation_test.go @@ -34,6 +34,18 @@ func TestFlexDaemonActivationPreflightUsesFlexAssetsWithoutMutation(t *testing.T } } +func TestFlexDaemonActivationWithoutAppliedStateSkipsNspawnSynchronization(t *testing.T) { + t.Parallel() + + service := &flexDaemonActivationService{ + log: slog.Default(), + state: &testStateStore{}, + } + if err := service.synchronizeActiveNspawn(t.Context(), "/unused/host/binary"); err != nil { + t.Fatalf("synchronizeActiveNspawn: %v", err) + } +} + func TestFlexDaemonActivationPreflightRejectsMachineOperationSignal(t *testing.T) { t.Parallel() diff --git a/scripts/install.sh b/scripts/install.sh index 9653fcb5..1ce52a65 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -254,11 +254,7 @@ download_binary() { install_binary() { local binary_path="$1" - local managed_dir="/usr/local/lib/aks-flex-node" - local current_path="${managed_dir}/aks-flex-node-current" - local blue_path="${managed_dir}/aks-flex-node-blue" - local green_path="${managed_dir}/aks-flex-node-green" - local last_good_path="${managed_dir}/aks-flex-node-last-good" + local current_path="/usr/local/lib/aks-flex-node/aks-flex-node-current" local compatibility_path="${INSTALL_DIR}/aks-flex-node" local candidate @@ -271,31 +267,14 @@ install_binary() { install -o root -g root -m 0755 "$binary_path" "$candidate" if [[ -e "$current_path" || -L "$current_path" ]]; then - if systemctl is-active --quiet aks-flex-node-agent.service 2>/dev/null; then - log_info "Activating candidate through the managed blue-green layout..." - if ! "$candidate" agent-upgrade; then - rm -f "$candidate" - log_error "Failed to activate the installed AKS Flex Node candidate" - return 1 - fi + log_info "Activating candidate through the managed agent layout..." + if ! "$candidate" agent-upgrade; then rm -f "$candidate" - log_success "Binary activated through $current_path" - return 0 + log_error "Failed to activate the installed AKS Flex Node candidate" + return 1 fi - - # Reset/unjoin removes daemon state and service assets but may leave the - # binary slots. With no running daemon to preserve, safely reseed the - # managed layout instead of writing through compatibility symlinks. - log_info "Reseeding inactive managed binary layout..." - install -d -o root -g root -m 0750 "$managed_dir" - rm -f "$compatibility_path" "$current_path" "$last_good_path" "$blue_path" "$green_path" - install -o root -g root -m 0755 "$candidate" "$blue_path" - ln -s "$blue_path" "$current_path" - ln -s "$blue_path" "$last_good_path" - install -d -o root -g root -m 0755 "$INSTALL_DIR" - ln -s "$current_path" "$compatibility_path" rm -f "$candidate" - log_success "Binary layout reseeded at $current_path" + log_success "Binary activated through the managed agent layout" return 0 fi From ea5e2b2767912718ff1913e5fc56422807040be2 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 01:37:02 +0000 Subject: [PATCH 31/45] fix: preserve inactive service during activation --- pkg/daemon/host_agent_activation.go | 45 ++++++++++++++++-------- pkg/daemon/host_agent_activation_test.go | 13 +++++-- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/pkg/daemon/host_agent_activation.go b/pkg/daemon/host_agent_activation.go index 6661ade3..8c9c3a23 100644 --- a/pkg/daemon/host_agent_activation.go +++ b/pkg/daemon/host_agent_activation.go @@ -55,11 +55,13 @@ func hostAgentActivationOptions(paths agentUpgradePaths, candidatePath string) a } type flexDaemonActivationService struct { - log *slog.Logger - paths agentUpgradePaths - state stateStore - systemdDir string - recoveryScript string + log *slog.Logger + paths agentUpgradePaths + state stateStore + systemdDir string + recoveryScript string + isServiceActive func(context.Context, *slog.Logger, string) bool + serviceWasActive bool } func newFlexDaemonActivationService(log *slog.Logger) (*flexDaemonActivationService, agentUpgradePaths, error) { @@ -72,15 +74,21 @@ func newFlexDaemonActivationService(log *slog.Logger) (*flexDaemonActivationServ } paths := defaultAgentUpgradePaths() return &flexDaemonActivationService{ - log: log, - paths: paths, - state: state, - systemdDir: systemdSystemDir, - recoveryScript: recoveryScriptPath, + log: log, + paths: paths, + state: state, + systemdDir: systemdSystemDir, + recoveryScript: recoveryScriptPath, + isServiceActive: utilexec.IsServiceActive, }, paths, nil } -func (s *flexDaemonActivationService) Preflight(_ context.Context, currentBinaryPath string) (agentbinary.ServicePlan, error) { +func (s *flexDaemonActivationService) Preflight(ctx context.Context, currentBinaryPath string) (agentbinary.ServicePlan, error) { + isActive := s.isServiceActive + if isActive == nil { + isActive = utilexec.IsServiceActive + } + s.serviceWasActive = isActive(ctx, s.log, ServiceUnitName) if _, err := os.Stat(s.paths.SignalPath); err == nil { return agentbinary.ServicePlan{}, fmt.Errorf("AgentUpgrade MachineOperation signal exists at %s", s.paths.SignalPath) } else if !errors.Is(err, os.ErrNotExist) { @@ -113,16 +121,20 @@ func (s *flexDaemonActivationService) Reload(ctx context.Context) error { } func (s *flexDaemonActivationService) Restart(ctx context.Context) error { + if !s.serviceWasActive { + s.log.Info("leaving inactive host agent service stopped after activation") + return nil + } if err := utilexec.RunCmd(ctx, s.log, utilexec.Systemctl(), "restart", ServiceUnitName); err != nil { return fmt.Errorf("systemctl restart %s: %w", ServiceUnitName, err) } return nil } -// WaitHealthy uses Flex's service and persisted active nspawn side. When a -// side exists, its exec-credential binary is synchronized only after systemd -// is stably running the expected host binary; reset hosts have no side to sync. -// Shared rollback calls this again with last-good. +// WaitHealthy preserves an inactive service during reset/reinstall. Otherwise, +// it synchronizes the active nspawn exec credential only after systemd is +// stably running the expected host binary. Shared rollback calls this again +// with last-good. func (s *flexDaemonActivationService) WaitHealthy(ctx context.Context, expectedBinaryPath string) error { healthCtx, cancel := context.WithTimeout(ctx, hostAgentHealthTimeout) defer cancel() @@ -130,6 +142,9 @@ func (s *flexDaemonActivationService) WaitHealthy(ctx context.Context, expectedB if err != nil { return fmt.Errorf("resolve expected daemon binary: %w", err) } + if !s.serviceWasActive { + return s.synchronizeActiveNspawn(healthCtx, expected) + } var healthySince time.Time ticker := time.NewTicker(hostAgentHealthPoll) defer ticker.Stop() diff --git a/pkg/daemon/host_agent_activation_test.go b/pkg/daemon/host_agent_activation_test.go index 3f306d7a..3f7d044c 100644 --- a/pkg/daemon/host_agent_activation_test.go +++ b/pkg/daemon/host_agent_activation_test.go @@ -34,15 +34,22 @@ func TestFlexDaemonActivationPreflightUsesFlexAssetsWithoutMutation(t *testing.T } } -func TestFlexDaemonActivationWithoutAppliedStateSkipsNspawnSynchronization(t *testing.T) { +func TestFlexDaemonActivationLeavesInactiveResetHostStopped(t *testing.T) { t.Parallel() + binary := filepath.Join(t.TempDir(), "candidate") + if err := os.WriteFile(binary, []byte("candidate"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } service := &flexDaemonActivationService{ log: slog.Default(), state: &testStateStore{}, } - if err := service.synchronizeActiveNspawn(t.Context(), "/unused/host/binary"); err != nil { - t.Fatalf("synchronizeActiveNspawn: %v", err) + if err := service.Restart(t.Context()); err != nil { + t.Fatalf("Restart: %v", err) + } + if err := service.WaitHealthy(t.Context(), binary); err != nil { + t.Fatalf("WaitHealthy: %v", err) } } From aeeb248a3ce227424f42deb4058d3321c1fce098 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 02:06:31 +0000 Subject: [PATCH 32/45] docs: clarify trusted upgrade transport contract --- pkg/daemon/agent_upgrade_binary.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 80f66676..29fd9dd7 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -194,6 +194,11 @@ func expectedAgentArchiveMember() (string, error) { } } +// secureAgentInstallOptions intentionally follows the merged Unbounded +// MachineOperation contract: HTTP transport and an omitted digest are allowed +// when the control plane trusts the archive source and transport path. Flex +// still bounds the archive, requires one exact member, and verifies the +// candidate executable. Production callers should supply HTTPS and SHA-256. func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.InstallOptions, error) { parsedURL, err := url.ParseRequestURI(strings.TrimSpace(rawURL)) if err != nil || parsedURL.Scheme != "http" && parsedURL.Scheme != "https" || parsedURL.Host == "" || parsedURL.User != nil || parsedURL.Fragment != "" { From efbfefba68b1f8329541bb0e4a94fe5b10189186 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 03:54:18 +0000 Subject: [PATCH 33/45] refactor: centralize release artifact naming --- pkg/daemon/agent_upgrade_binary.go | 12 ++----- pkg/daemon/agent_upgrade_binary_test.go | 10 ++++-- pkg/release/artifacts.go | 21 +++++++++++ pkg/release/artifacts_test.go | 47 +++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 pkg/release/artifacts.go create mode 100644 pkg/release/artifacts_test.go diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 29fd9dd7..02c4a785 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -16,6 +16,7 @@ import ( "syscall" "time" + "github.com/Azure/AKSFlexNode/pkg/release" "github.com/Azure/AKSFlexNode/pkg/utils/utilio" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -185,15 +186,6 @@ func replaceSymlink(linkPath, targetPath string) error { return os.Rename(tempPath, linkPath) } -func expectedAgentArchiveMember() (string, error) { - switch runtime.GOARCH { - case "amd64", "arm64": - return "aks-flex-node-linux-" + runtime.GOARCH, nil - default: - return "", fmt.Errorf("unsupported agent upgrade architecture %q", runtime.GOARCH) - } -} - // secureAgentInstallOptions intentionally follows the merged Unbounded // MachineOperation contract: HTTP transport and an omitted digest are allowed // when the control plane trusts the archive source and transport path. Flex @@ -211,7 +203,7 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta return agentbinary.InstallOptions{}, fmt.Errorf("expected SHA-256 must be exactly 64 hexadecimal characters") } } - member, err := expectedAgentArchiveMember() + member, err := release.AgentBinaryArchiveMember(runtime.GOOS, runtime.GOARCH) if err != nil { return agentbinary.InstallOptions{}, err } diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go index 01440c76..244f8f00 100644 --- a/pkg/daemon/agent_upgrade_binary_test.go +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -7,6 +7,8 @@ import ( "runtime" "strings" "testing" + + "github.com/Azure/AKSFlexNode/pkg/release" ) func TestEnsureAgentUpgradeLayoutMigratesLegacyBinaryIdempotently(t *testing.T) { @@ -69,8 +71,12 @@ func TestSecureAgentInstallOptions(t *testing.T) { if err != nil { t.Fatalf("secureAgentInstallOptions: %v", err) } - if opts.ExpectedMember != "aks-flex-node-linux-"+runtime.GOARCH { - t.Fatalf("ExpectedMember = %q", opts.ExpectedMember) + wantMember, err := release.AgentBinaryArchiveMember(runtime.GOOS, runtime.GOARCH) + if err != nil { + t.Fatalf("AgentBinaryArchiveMember: %v", err) + } + if opts.ExpectedMember != wantMember { + t.Fatalf("ExpectedMember = %q, want %q", opts.ExpectedMember, wantMember) } if opts.MaxArchiveBytes != agentUpgradeMaxArchiveBytes || opts.MaxExtractedBytes != agentUpgradeMaxBinaryBytes { t.Fatalf("size limits = %d, %d", opts.MaxArchiveBytes, opts.MaxExtractedBytes) diff --git a/pkg/release/artifacts.go b/pkg/release/artifacts.go new file mode 100644 index 00000000..1b5879fd --- /dev/null +++ b/pkg/release/artifacts.go @@ -0,0 +1,21 @@ +// Package release defines the artifact naming contract shared by Flex Node +// release producers and consumers. +package release + +import "fmt" + +const AgentBinaryBaseName = "aks-flex-node" + +// AgentBinaryArchiveMember returns the binary member name used by a release +// archive for the requested platform. +func AgentBinaryArchiveMember(goos, goarch string) (string, error) { + if goos != "linux" { + return "", fmt.Errorf("unsupported agent release operating system %q", goos) + } + switch goarch { + case "amd64", "arm64": + return AgentBinaryBaseName + "-" + goos + "-" + goarch, nil + default: + return "", fmt.Errorf("unsupported agent release architecture %q", goarch) + } +} diff --git a/pkg/release/artifacts_test.go b/pkg/release/artifacts_test.go new file mode 100644 index 00000000..36362cff --- /dev/null +++ b/pkg/release/artifacts_test.go @@ -0,0 +1,47 @@ +package release + +import "testing" + +func TestAgentBinaryArchiveMember(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + goos string + goarch string + want string + wantErr bool + }{ + "amd64": { + goos: "linux", + goarch: "amd64", + want: "aks-flex-node-linux-amd64", + }, + "arm64": { + goos: "linux", + goarch: "arm64", + want: "aks-flex-node-linux-arm64", + }, + "unsupported operating system": { + goos: "windows", + goarch: "amd64", + wantErr: true, + }, + "unsupported architecture": { + goos: "linux", + goarch: "riscv64", + wantErr: true, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + got, err := AgentBinaryArchiveMember(tt.goos, tt.goarch) + if (err != nil) != tt.wantErr { + t.Fatalf("AgentBinaryArchiveMember() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Fatalf("AgentBinaryArchiveMember() = %q, want %q", got, tt.want) + } + }) + } +} From 31d5d3a6c1367f4a21e57f4861c352d73c8ee185 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:15:33 +0000 Subject: [PATCH 34/45] fix: serialize startup upgrade recovery --- pkg/daemon/agent_upgrade.go | 15 --------------- pkg/daemon/daemon.go | 21 +++++++-------------- 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 6599f563..6016cb6d 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -387,21 +387,6 @@ func RecoverAgentUpgrade(ctx context.Context, message string) error { return ctx.Err() } -func retryAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c client.Client, executor *hostAgentUpgradeExecutor) { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if err := publishAndClearAgentUpgradeSignal(ctx, log, c, executor); err != nil && ctx.Err() == nil { - log.Warn("failed to publish durable AgentUpgrade result; will retry", "error", err) - } - } - } -} - func publishAndClearAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c client.Client, executor *hostAgentUpgradeExecutor) error { paths := executor.paths signals := executor.signals diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 79866867..3c02a866 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -109,22 +109,15 @@ func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { return fmt.Errorf("setup daemon controller: %w", err) } - publishCtx, stopPublisher := context.WithCancel(ctx) - defer stopPublisher() - go func() { - // Success is only safe to publish after manager startup has reached cache - // readiness. If startup fails, systemd retains the signal and recovers. - if !mgr.GetCache().WaitForCacheSync(publishCtx) { - return - } - if err := publishAndClearAgentUpgradeSignal(publishCtx, log, directClient, upgrades); err != nil { - log.Warn("failed to publish AgentUpgrade startup result", "error", err) - } - retryAgentUpgradeSignal(publishCtx, log, directClient, upgrades) - }() + // Publish durable upgrade recovery before starting the serialized controller, + // matching Unbounded's startup ordering. This prevents recovery-time host and + // nspawn mutation from racing repave or reset reconciliation. + if err := publishAndClearAgentUpgradeSignal(ctx, log, directClient, upgrades); err != nil { + // Retain the signal so a later daemon start can retry publication. + log.Warn("failed to publish AgentUpgrade startup result", "error", err) + } err = mgr.Start(ctx) - stopPublisher() repaves.log.Info("daemon shutting down") return err } From e27364c767def7fa4d4afd34c0b0c38d91cd0323 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:18:45 +0000 Subject: [PATCH 35/45] refactor: keep artifact naming in config --- pkg/{release => config}/artifacts.go | 5 ++--- pkg/{release => config}/artifacts_test.go | 2 +- pkg/daemon/agent_upgrade_binary.go | 4 ++-- pkg/daemon/agent_upgrade_binary_test.go | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) rename pkg/{release => config}/artifacts.go (80%) rename pkg/{release => config}/artifacts_test.go (98%) diff --git a/pkg/release/artifacts.go b/pkg/config/artifacts.go similarity index 80% rename from pkg/release/artifacts.go rename to pkg/config/artifacts.go index 1b5879fd..ee88867a 100644 --- a/pkg/release/artifacts.go +++ b/pkg/config/artifacts.go @@ -1,9 +1,8 @@ -// Package release defines the artifact naming contract shared by Flex Node -// release producers and consumers. -package release +package config import "fmt" +// AgentBinaryBaseName is the installed command and release artifact base name. const AgentBinaryBaseName = "aks-flex-node" // AgentBinaryArchiveMember returns the binary member name used by a release diff --git a/pkg/release/artifacts_test.go b/pkg/config/artifacts_test.go similarity index 98% rename from pkg/release/artifacts_test.go rename to pkg/config/artifacts_test.go index 36362cff..e7c8b673 100644 --- a/pkg/release/artifacts_test.go +++ b/pkg/config/artifacts_test.go @@ -1,4 +1,4 @@ -package release +package config import "testing" diff --git a/pkg/daemon/agent_upgrade_binary.go b/pkg/daemon/agent_upgrade_binary.go index 02c4a785..41cbca66 100644 --- a/pkg/daemon/agent_upgrade_binary.go +++ b/pkg/daemon/agent_upgrade_binary.go @@ -16,7 +16,7 @@ import ( "syscall" "time" - "github.com/Azure/AKSFlexNode/pkg/release" + "github.com/Azure/AKSFlexNode/pkg/config" "github.com/Azure/AKSFlexNode/pkg/utils/utilio" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -203,7 +203,7 @@ func secureAgentInstallOptions(rawURL, expectedDigest string) (agentbinary.Insta return agentbinary.InstallOptions{}, fmt.Errorf("expected SHA-256 must be exactly 64 hexadecimal characters") } } - member, err := release.AgentBinaryArchiveMember(runtime.GOOS, runtime.GOARCH) + member, err := config.AgentBinaryArchiveMember(runtime.GOOS, runtime.GOARCH) if err != nil { return agentbinary.InstallOptions{}, err } diff --git a/pkg/daemon/agent_upgrade_binary_test.go b/pkg/daemon/agent_upgrade_binary_test.go index 244f8f00..02f625bb 100644 --- a/pkg/daemon/agent_upgrade_binary_test.go +++ b/pkg/daemon/agent_upgrade_binary_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/Azure/AKSFlexNode/pkg/release" + "github.com/Azure/AKSFlexNode/pkg/config" ) func TestEnsureAgentUpgradeLayoutMigratesLegacyBinaryIdempotently(t *testing.T) { @@ -71,7 +71,7 @@ func TestSecureAgentInstallOptions(t *testing.T) { if err != nil { t.Fatalf("secureAgentInstallOptions: %v", err) } - wantMember, err := release.AgentBinaryArchiveMember(runtime.GOOS, runtime.GOARCH) + wantMember, err := config.AgentBinaryArchiveMember(runtime.GOOS, runtime.GOARCH) if err != nil { t.Fatalf("AgentBinaryArchiveMember: %v", err) } From e87adb215dfe1b704052e6331f82d1968a1d0f98 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:20:13 +0000 Subject: [PATCH 36/45] refactor: delegate installer activation directly --- scripts/install.sh | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 1ce52a65..e6464ea0 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -256,31 +256,36 @@ install_binary() { local binary_path="$1" local current_path="/usr/local/lib/aks-flex-node/aks-flex-node-current" local compatibility_path="${INSTALL_DIR}/aks-flex-node" - local candidate + local candidate_path active_path log_info "Installing binary to $INSTALL_DIR..." - # Always stage separately. Installing directly through BinaryPath after - # migration would dereference its symlink and overwrite an active or - # last-good slot in place. - candidate=$(mktemp /var/tmp/aks-flex-node-install-candidate.XXXXXX) - install -o root -g root -m 0755 "$binary_path" "$candidate" - if [[ -e "$current_path" || -L "$current_path" ]]; then - log_info "Activating candidate through the managed agent layout..." - if ! "$candidate" agent-upgrade; then - rm -f "$candidate" + candidate_path=$(readlink -f "$binary_path") + active_path=$(readlink -f "$current_path") + if [[ ! -f "$candidate_path" || ! -x "$candidate_path" ]]; then + log_error "Managed installation requires a separately staged executable candidate" + return 1 + fi + if [[ "$candidate_path" == "$active_path" ]]; then + log_error "Candidate must be staged separately from the active AKS Flex Node binary" + return 1 + fi + + # Candidate delivery is the installer's only responsibility here. The + # Go activation command owns verification, switching, service handling, + # health checks, and rollback for an existing managed installation. + log_info "Delegating managed binary activation to the candidate..." + if ! "$candidate_path" agent-upgrade; then log_error "Failed to activate the installed AKS Flex Node candidate" return 1 fi - rm -f "$candidate" log_success "Binary activated through the managed agent layout" return 0 fi install -d -o root -g root -m 0755 "$INSTALL_DIR" - install -o root -g root -m 0755 "$candidate" "$compatibility_path" - rm -f "$candidate" + install -o root -g root -m 0755 "$binary_path" "$compatibility_path" log_success "Binary installed to $compatibility_path" } From e7c08495cb0229c598025d3d5850a11ed8fdb72a Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:23:53 +0000 Subject: [PATCH 37/45] test: activate staged candidate during rejoin --- hack/e2e/README.md | 2 +- hack/e2e/lib/node-join.sh | 15 +++++++++++---- scripts/install.sh | 28 ++++++---------------------- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/hack/e2e/README.md b/hack/e2e/README.md index 34d88bad..9e8e85f2 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -144,7 +144,7 @@ its Azure resource name remains lowercase. This verifies that an omitted `agent.nodeName` is derived from the normalized hostname and still joins the cluster under the lowercase VM name. -Each join path uploads the locally built binary, renders a config file, installs the binary through `scripts/install.sh` with `AKS_FLEX_NODE_LOCAL_BINARY`, and starts the node through a transient systemd unit. The installed agent service is then validated with systemd checks. +Each join path uploads the locally built binary and renders a config file. Fresh hosts install it through `scripts/install.sh` with `AKS_FLEX_NODE_LOCAL_BINARY`; rejoin hosts with an existing managed layout invoke the uploaded candidate's `agent-upgrade` command before bootstrap. The node starts through a transient systemd unit, and the installed agent service is then validated with systemd checks. ## Agent Upgrade Validation diff --git a/hack/e2e/lib/node-join.sh b/hack/e2e/lib/node-join.sh index e92499a9..b2948d55 100755 --- a/hack/e2e/lib/node-join.sh +++ b/hack/e2e/lib/node-join.sh @@ -39,10 +39,17 @@ _deploy_and_start_agent() { remote_exec "${vm_ip}" "UNIT_NAME=${unit_name} E2E_NODE_JOIN_TIMEOUT=${E2E_NODE_JOIN_TIMEOUT} E2E_KUBERNETES_VERSION=${E2E_KUBERNETES_VERSION} bash -s" <<'REMOTE' set -euo pipefail -sudo AKS_FLEX_NODE_LOCAL_BINARY=/tmp/aks-flex-node-binary \ - AKS_FLEX_NODE_VERSION=e2e-local \ - SKIP_AZCLI=true \ - bash /tmp/aks-flex-node-install.sh --yes +managed_current=/usr/local/lib/aks-flex-node/aks-flex-node-current +if [[ -e "${managed_current}" || -L "${managed_current}" ]]; then + echo "Existing managed layout found; activating the separately staged E2E candidate..." + sudo chmod 0755 /tmp/aks-flex-node-binary + sudo /tmp/aks-flex-node-binary agent-upgrade +else + sudo AKS_FLEX_NODE_LOCAL_BINARY=/tmp/aks-flex-node-binary \ + AKS_FLEX_NODE_VERSION=e2e-local \ + SKIP_AZCLI=true \ + bash /tmp/aks-flex-node-install.sh --yes +fi sudo /usr/local/bin/aks-flex-node version diff --git a/scripts/install.sh b/scripts/install.sh index e6464ea0..aa4a40f8 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -256,32 +256,16 @@ install_binary() { local binary_path="$1" local current_path="/usr/local/lib/aks-flex-node/aks-flex-node-current" local compatibility_path="${INSTALL_DIR}/aks-flex-node" - local candidate_path active_path log_info "Installing binary to $INSTALL_DIR..." + # This script owns only fresh installation. Managed updates must execute a + # separately delivered candidate so the Go activation flow preserves the + # active and last-good binaries. if [[ -e "$current_path" || -L "$current_path" ]]; then - candidate_path=$(readlink -f "$binary_path") - active_path=$(readlink -f "$current_path") - if [[ ! -f "$candidate_path" || ! -x "$candidate_path" ]]; then - log_error "Managed installation requires a separately staged executable candidate" - return 1 - fi - if [[ "$candidate_path" == "$active_path" ]]; then - log_error "Candidate must be staged separately from the active AKS Flex Node binary" - return 1 - fi - - # Candidate delivery is the installer's only responsibility here. The - # Go activation command owns verification, switching, service handling, - # health checks, and rollback for an existing managed installation. - log_info "Delegating managed binary activation to the candidate..." - if ! "$candidate_path" agent-upgrade; then - log_error "Failed to activate the installed AKS Flex Node candidate" - return 1 - fi - log_success "Binary activated through the managed agent layout" - return 0 + log_error "A managed AKS Flex Node installation already exists" + log_error "Run the separately staged candidate with: agent-upgrade" + return 1 fi install -d -o root -g root -m 0755 "$INSTALL_DIR" From fc22fd819cef9e03420a127bdc6e33780fb48757 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:26:23 +0000 Subject: [PATCH 38/45] revert: keep install script unchanged --- scripts/install.sh | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index aa4a40f8..5fc7a457 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -254,24 +254,15 @@ download_binary() { install_binary() { local binary_path="$1" - local current_path="/usr/local/lib/aks-flex-node/aks-flex-node-current" - local compatibility_path="${INSTALL_DIR}/aks-flex-node" log_info "Installing binary to $INSTALL_DIR..." - # This script owns only fresh installation. Managed updates must execute a - # separately delivered candidate so the Go activation flow preserves the - # active and last-good binaries. - if [[ -e "$current_path" || -L "$current_path" ]]; then - log_error "A managed AKS Flex Node installation already exists" - log_error "Run the separately staged candidate with: agent-upgrade" - return 1 - fi - - install -d -o root -g root -m 0755 "$INSTALL_DIR" - install -o root -g root -m 0755 "$binary_path" "$compatibility_path" + # Install binary + cp "$binary_path" "$INSTALL_DIR/aks-flex-node" + chmod +x "$INSTALL_DIR/aks-flex-node" + chown root:root "$INSTALL_DIR/aks-flex-node" - log_success "Binary installed to $compatibility_path" + log_success "Binary installed to $INSTALL_DIR/aks-flex-node" } warn_install_dir_not_in_path() { From 4111c54bfed9838b6339a240c38882285679af87 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:28:54 +0000 Subject: [PATCH 39/45] fix: make rollback conditional and recovery durable --- pkg/daemon/agent_upgrade.go | 53 +++++++++++---------- pkg/daemon/agent_upgrade_test.go | 31 ++++++++++++ pkg/daemon/assets/aks-flex-node-recovery.sh | 8 ++-- pkg/daemon/lifecycle_test.go | 2 + 4 files changed, 66 insertions(+), 28 deletions(-) diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 6016cb6d..83d7221b 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -306,23 +306,11 @@ func (e *hostAgentUpgradeExecutor) rollback(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - lastGood, err := resolvedExecutable(e.paths.LastGoodPath) - if err != nil { - return fmt.Errorf("resolve last-good agent binary: %w", err) - } - if err := replaceSymlink(e.paths.CurrentPath, lastGood); err != nil { - return fmt.Errorf("restore last-good agent binary: %w", err) - } signal, err := e.signals.read() if err != nil { return err } - if signal != nil && signal.ActiveMachine != "" { - if err := synchronizeNspawnAgentBinary(lastGood, signal.ActiveMachine); err != nil { - return fmt.Errorf("restore active nspawn agent binary: %w", err) - } - } - return nil + return rollbackAgentUpgradeFiles(e.paths, signal) } func (e *hostAgentUpgradeExecutor) Restart(ctx context.Context) error { @@ -368,21 +356,12 @@ func RecoverAgentUpgrade(ctx context.Context, message string) error { if err := signals.recordFailure(message); err != nil { return err } - lastGood, err := resolvedExecutable(paths.LastGoodPath) - if err != nil { - return fmt.Errorf("resolve last-good agent binary: %w", err) - } - if err := replaceSymlink(paths.CurrentPath, lastGood); err != nil { - return fmt.Errorf("restore last-good agent binary: %w", err) - } signal, err := signals.read() if err != nil { return err } - if signal != nil && signal.ActiveMachine != "" { - if err := synchronizeNspawnAgentBinary(lastGood, signal.ActiveMachine); err != nil { - return err - } + if err := rollbackAgentUpgradeFiles(paths, signal); err != nil { + return err } return ctx.Err() } @@ -405,7 +384,13 @@ func publishAndClearAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c if signal.Failure == "" { if validationErr := validateStartedAgentUpgrade(paths, signal); validationErr != nil { signal.Failure = validationErr.Error() - signal.RecoveryRequired = true + candidateActive, activeErr := agentUpgradeCandidateIsActive(paths, signal) + if activeErr != nil { + signal.Failure = errors.Join(validationErr, activeErr).Error() + signal.RecoveryRequired = true + } else { + signal.RecoveryRequired = candidateActive + } if err := signals.write(*signal); err != nil { return err } @@ -513,7 +498,25 @@ func validateStartedAgentUpgrade(paths agentUpgradePaths, signal *agentUpgradeSi return nil } +func agentUpgradeCandidateIsActive(paths agentUpgradePaths, signal *agentUpgradeSignal) (bool, error) { + if signal == nil || signal.CandidatePath != paths.BluePath && signal.CandidatePath != paths.GreenPath { + return false, nil + } + current, err := resolvedExecutable(paths.CurrentPath) + if err != nil { + return false, fmt.Errorf("resolve current agent binary for rollback: %w", err) + } + return current == signal.CandidatePath, nil +} + func rollbackAgentUpgradeFiles(paths agentUpgradePaths, signal *agentUpgradeSignal) error { + candidateActive, err := agentUpgradeCandidateIsActive(paths, signal) + if err != nil { + return err + } + if !candidateActive { + return nil + } lastGood, err := resolvedExecutable(paths.LastGoodPath) if err != nil { return fmt.Errorf("resolve last-good agent binary: %w", err) diff --git a/pkg/daemon/agent_upgrade_test.go b/pkg/daemon/agent_upgrade_test.go index cec9d398..bb324b2f 100644 --- a/pkg/daemon/agent_upgrade_test.go +++ b/pkg/daemon/agent_upgrade_test.go @@ -214,6 +214,9 @@ func TestHostAgentUpgradeExecutorAbortUsesCleanupContext(t *testing.T) { if err := signals.recordPending("operation-1", "", "instance-1"); err != nil { t.Fatalf("recordPending: %v", err) } + if err := signals.recordCandidate(paths.GreenPath); err != nil { + t.Fatalf("recordCandidate: %v", err) + } executor := &hostAgentUpgradeExecutor{paths: paths, signals: signals, instanceID: "instance-1"} ctx, cancel := context.WithCancel(t.Context()) cancel() @@ -234,6 +237,9 @@ func TestHostAgentUpgradeExecutorAbortPreservesSignalOnRollbackFailure(t *testin if err := signals.recordPending("operation-1", "", "instance-1"); err != nil { t.Fatalf("recordPending: %v", err) } + if err := signals.recordCandidate(paths.BluePath); err != nil { + t.Fatalf("recordCandidate: %v", err) + } executor := &hostAgentUpgradeExecutor{paths: paths, signals: signals, instanceID: "instance-1"} if err := executor.Abort(t.Context()); err == nil { t.Fatal("Abort error = nil") @@ -362,6 +368,31 @@ func TestPublishAgentUpgradeSignalIgnoresInitiatingProcess(t *testing.T) { } } +func TestRollbackAgentUpgradeFilesDoesNotDowngradeBeforeSwitch(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + for path, content := range map[string]string{paths.BluePath: "last-good", paths.GreenPath: "active"} { + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } + } + if err := os.Symlink(paths.GreenPath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.LastGoodPath); err != nil { + t.Fatalf("Symlink last-good: %v", err) + } + + if err := rollbackAgentUpgradeFiles(paths, &agentUpgradeSignal{CandidatePath: paths.BluePath}); err != nil { + t.Fatalf("rollbackAgentUpgradeFiles: %v", err) + } + assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) +} + func TestFilesHaveEqualSHA256(t *testing.T) { t.Parallel() diff --git a/pkg/daemon/assets/aks-flex-node-recovery.sh b/pkg/daemon/assets/aks-flex-node-recovery.sh index 71d3fa86..5cd2432d 100644 --- a/pkg/daemon/assets/aks-flex-node-recovery.sh +++ b/pkg/daemon/assets/aks-flex-node-recovery.sh @@ -13,7 +13,9 @@ if [[ -z "${last_good}" || ! -x "${last_good}" ]]; then exit 1 fi +status=0 "${last_good}" recover-agent-upgrade \ - --message "upgraded daemon failed repeatedly; restored last-good binary" -systemctl reset-failed aks-flex-node-agent.service -systemctl --no-block restart aks-flex-node-agent.service + --message "upgraded daemon failed repeatedly; restored last-good binary" || status=$? +systemctl reset-failed aks-flex-node-agent.service || status=$? +systemctl --no-block restart aks-flex-node-agent.service || status=$? +exit "${status}" diff --git a/pkg/daemon/lifecycle_test.go b/pkg/daemon/lifecycle_test.go index cd888658..d0d87e4d 100644 --- a/pkg/daemon/lifecycle_test.go +++ b/pkg/daemon/lifecycle_test.go @@ -91,6 +91,8 @@ func TestAgentServiceIncludesUpgradeRecovery(t *testing.T) { "aks-flex-node-last-good", "agent-upgrade-signal.json", "systemctl --no-block restart", + "|| status=$?", + "exit \"${status}\"", } { if !strings.Contains(script, expected) { t.Fatalf("recovery script does not contain %q", expected) From e2ca736214328008835c8d99c834d59270e13c2f Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 04:44:56 +0000 Subject: [PATCH 40/45] fix: gate reconciliation until upgrade readiness --- docs/usages/operations.md | 2 +- docs/usages/operator-first-boot.md | 12 +++- hack/e2e/lib/agent-upgrade.sh | 5 ++ pkg/daemon/agent_upgrade.go | 41 ++++++++---- pkg/daemon/agent_upgrade_test.go | 36 ++++++++-- pkg/daemon/daemon.go | 42 +++++++++--- pkg/daemon/lifecycle.go | 6 +- pkg/daemon/machineoperation_reconciler.go | 7 +- .../machineoperation_reconciler_test.go | 19 ++++++ pkg/daemon/startup_gate.go | 65 +++++++++++++++++++ pkg/daemon/startup_gate_test.go | 41 ++++++++++++ 11 files changed, 244 insertions(+), 32 deletions(-) create mode 100644 pkg/daemon/startup_gate.go create mode 100644 pkg/daemon/startup_gate_test.go diff --git a/docs/usages/operations.md b/docs/usages/operations.md index 3a7649fe..bfddcdab 100644 --- a/docs/usages/operations.md +++ b/docs/usages/operations.md @@ -81,7 +81,7 @@ A host provisioning system that has already authenticated and staged a candidate sudo /var/tmp/aks-flex-node-candidate agent-upgrade ``` -The candidate must be staged separately from the installed binary. Direct activation and `MachineOperation` activation share one host lock and refuse to overlap with a pending operation signal. Both paths verify the candidate, switch the same blue/green layout, restart `aks-flex-node-agent.service`, verify the running executable, synchronize the active nspawn exec-credential binary, and restore last-good on activation failure. +The candidate must be staged separately from the installed binary. Direct activation and `MachineOperation` activation share one host lock and refuse to overlap with a pending operation signal. Both paths verify the candidate, switch the same blue/green layout, and restore last-good on activation failure. If `aks-flex-node-agent.service` is active, direct activation restarts it, verifies the running executable, and synchronizes the active nspawn exec-credential binary. If the service is already inactive during reset/rejoin provisioning, activation preserves that stopped state; the subsequent bootstrap starts the service and worker. ## Nspawn Worker diff --git a/docs/usages/operator-first-boot.md b/docs/usages/operator-first-boot.md index a461db35..822a1d90 100644 --- a/docs/usages/operator-first-boot.md +++ b/docs/usages/operator-first-boot.md @@ -249,7 +249,7 @@ label. ## 3. Install temporary AKS Flex daemon RBAC > [!IMPORTANT] -> **Temporary preview requirement:** When the Machina MachineOperation CRD is +> **Temporary preview requirement:** When the Unbounded MachineOperation CRD is > installed, AKS Flex Node discovers it and enables its MachineOperation > reconciler. A future AKS RP release will install and manage the required > ClusterRole and ClusterRoleBinding automatically as part of FlexNodes pool @@ -287,7 +287,17 @@ rules: resources: - machineoperations/status verbs: + - get + - patch - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/hack/e2e/lib/agent-upgrade.sh b/hack/e2e/lib/agent-upgrade.sh index cbdaaeee..63a2d704 100644 --- a/hack/e2e/lib/agent-upgrade.sh +++ b/hack/e2e/lib/agent-upgrade.sh @@ -218,6 +218,11 @@ agent_upgrade_e2e() { validate_node_joined "${vm_name}" _agent_upgrade_ensure_api + # MachineOperation discovery occurs during daemon startup. Restart after the + # CRD is established so this focused command also works on previously joined + # environments where the API was absent. + remote_exec "${vm_ip}" 'sudo systemctl restart aks-flex-node-agent.service' + validate_node_joined "${vm_name}" _agent_upgrade_prepare_server "${vm_ip}" success_digest="$(_agent_upgrade_digest "${vm_ip}" success.tar.gz)" failure_digest="$(_agent_upgrade_digest "${vm_ip}" failure.tar.gz)" diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 83d7221b..c37c68d4 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -68,6 +68,7 @@ type agentUpgradeSignal struct { ActiveMachine string `json:"activeMachine,omitempty"` CandidatePath string `json:"candidatePath,omitempty"` InitiatingDaemonInstance string `json:"initiatingDaemonInstance,omitempty"` + SwitchCommitted bool `json:"switchCommitted,omitempty"` RecoveryRequired bool `json:"recoveryRequired,omitempty"` Failure string `json:"failure,omitempty"` } @@ -96,6 +97,18 @@ func (s agentUpgradeSignalStore) recordCandidate(candidatePath string) error { return s.write(*signal) } +func (s agentUpgradeSignalStore) recordSwitchCommitted() error { + signal, err := s.read() + if err != nil { + return err + } + if signal == nil { + return fmt.Errorf("no pending AgentUpgrade signal") + } + signal.SwitchCommitted = true + return s.write(*signal) +} + func (s agentUpgradeSignalStore) recordFailure(message string) error { signal, err := s.read() if err != nil { @@ -258,6 +271,9 @@ func (e *hostAgentUpgradeExecutor) Stage(ctx context.Context, request agentUpgra if err := installAndSwitchAgentBinary(ctx, e.log, request.downloadURL, request.sha256, e.paths); err != nil { return err } + if err := e.signals.recordSwitchCommitted(); err != nil { + return e.rollbackAfterStage(ctx, fmt.Errorf("record committed AgentUpgrade switch: %w", err)) + } signal, err := e.signals.read() if err != nil { @@ -412,11 +428,6 @@ func publishAndClearAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c } } - finishOperation := executor.finishMachineOperation - if finishOperation == nil { - finishOperation = agentdaemon.FinishMachineOperation - } - finishErr := finishOperation(ctx, c, agentdaemon.MachineOperation{Name: signal.OperationName}, result) if signal.RecoveryRequired { lastGood, err := resolvedExecutable(paths.LastGoodPath) if err != nil { @@ -434,16 +445,19 @@ func publishAndClearAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) restartErr := executor.Restart(cleanupCtx) cancel() - if finishErr != nil || restartErr != nil { - return errors.Join( - wrapOptionalError("publish AgentUpgrade result", finishErr), - wrapOptionalError("restart last-good agent", restartErr), - ) + if restartErr != nil { + return fmt.Errorf("restart last-good agent: %w", restartErr) } - // Keep the signal until the last-good process confirms it is running. + // Keep the signal and terminal status unpublished until the last-good + // process confirms that it is running. return nil } } + finishOperation := executor.finishMachineOperation + if finishOperation == nil { + finishOperation = agentdaemon.FinishMachineOperation + } + finishErr := finishOperation(ctx, c, agentdaemon.MachineOperation{Name: signal.OperationName}, result) if finishErr != nil { return fmt.Errorf("publish AgentUpgrade result: %w", finishErr) } @@ -510,11 +524,14 @@ func agentUpgradeCandidateIsActive(paths agentUpgradePaths, signal *agentUpgrade } func rollbackAgentUpgradeFiles(paths agentUpgradePaths, signal *agentUpgradeSignal) error { + if signal == nil { + return nil + } candidateActive, err := agentUpgradeCandidateIsActive(paths, signal) if err != nil { return err } - if !candidateActive { + if !signal.SwitchCommitted && !candidateActive { return nil } lastGood, err := resolvedExecutable(paths.LastGoodPath) diff --git a/pkg/daemon/agent_upgrade_test.go b/pkg/daemon/agent_upgrade_test.go index bb324b2f..d7de89de 100644 --- a/pkg/daemon/agent_upgrade_test.go +++ b/pkg/daemon/agent_upgrade_test.go @@ -112,6 +112,9 @@ func TestAgentUpgradeSignalStoreLifecycle(t *testing.T) { if err := store.recordCandidate("/slots/green"); err != nil { t.Fatalf("recordCandidate: %v", err) } + if err := store.recordSwitchCommitted(); err != nil { + t.Fatalf("recordSwitchCommitted: %v", err) + } if err := store.recordFailure("rolled back"); err != nil { t.Fatalf("recordFailure: %v", err) } @@ -119,7 +122,7 @@ func TestAgentUpgradeSignalStoreLifecycle(t *testing.T) { if err != nil { t.Fatalf("read: %v", err) } - if signal == nil || signal.OperationName != "operation-1" || signal.ActiveMachine != "kube1" || signal.CandidatePath != "/slots/green" || signal.InitiatingDaemonInstance != "instance-1" || signal.Failure != "rolled back" || !signal.RecoveryRequired { + if signal == nil || signal.OperationName != "operation-1" || signal.ActiveMachine != "kube1" || signal.CandidatePath != "/slots/green" || signal.InitiatingDaemonInstance != "instance-1" || !signal.SwitchCommitted || signal.Failure != "rolled back" || !signal.RecoveryRequired { t.Fatalf("signal = %#v", signal) } info, err := os.Stat(path) @@ -334,8 +337,8 @@ func TestPublishAgentUpgradeFailureRestartsIntoLastGoodBeforeClearingSignal(t *t if signal, err := signals.read(); err != nil || signal == nil { t.Fatalf("signal cleared before last-good startup: %#v, %v", signal, err) } - if restarts != 1 || finished != 1 { - t.Fatalf("restarts = %d, finished = %d", restarts, finished) + if restarts != 1 || finished != 0 { + t.Fatalf("restarts = %d, finished = %d; terminal status must wait for last-good", restarts, finished) } executor.instanceID = "last-good-instance" @@ -346,7 +349,7 @@ func TestPublishAgentUpgradeFailureRestartsIntoLastGoodBeforeClearingSignal(t *t if signal, err := signals.read(); err != nil || signal != nil { t.Fatalf("signal after last-good startup = %#v, %v", signal, err) } - if restarts != 1 || finished != 2 { + if restarts != 1 || finished != 1 { t.Fatalf("restarts = %d, finished = %d", restarts, finished) } } @@ -393,6 +396,31 @@ func TestRollbackAgentUpgradeFilesDoesNotDowngradeBeforeSwitch(t *testing.T) { assertResolvedPath(t, paths.CurrentPath, paths.GreenPath) } +func TestRollbackAgentUpgradeFilesRetriesAfterHostLinkWasRestored(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + for _, path := range []string{paths.BluePath, paths.GreenPath} { + if err := os.WriteFile(path, []byte(path), 0o755); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } + } + if err := os.Symlink(paths.GreenPath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + + err := rollbackAgentUpgradeFiles(paths, &agentUpgradeSignal{ + CandidatePath: paths.BluePath, + SwitchCommitted: true, + }) + if err == nil || !strings.Contains(err.Error(), "resolve last-good") { + t.Fatalf("rollbackAgentUpgradeFiles error = %v, want retry to resolve last-good", err) + } +} + func TestFilesHaveEqualSHA256(t *testing.T) { t.Parallel() diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 3c02a866..a3c8311b 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -25,8 +25,9 @@ import ( ) const ( - daemonCredentialDir = "daemon-credentials" //nolint:gosec // Directory name, not a credential. - daemonCredentialGroup = "aks-flex-node-daemons" //nolint:gosec // Kubernetes group name, not a credential. + agentUpgradeStartupStability = 3 * time.Second + daemonCredentialDir = "daemon-credentials" //nolint:gosec // Directory name, not a credential. + daemonCredentialGroup = "aks-flex-node-daemons" //nolint:gosec // Kubernetes group name, not a credential. ) // Run starts the machine-driven daemon loop. @@ -105,17 +106,38 @@ func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { if err != nil { return err } - if err := daemon.SetupController("aks-flex-node-daemon", mgr, machineOperations, repaves); err != nil { + gate := newStartupGate() + if err := daemon.SetupController( + "aks-flex-node-daemon", + mgr, + gatedMachineOperationReconciler{delegate: machineOperations, gate: gate}, + gatedRepaveReconciler{delegate: repaves, gate: gate}, + ); err != nil { return fmt.Errorf("setup daemon controller: %w", err) } - // Publish durable upgrade recovery before starting the serialized controller, - // matching Unbounded's startup ordering. This prevents recovery-time host and - // nspawn mutation from racing repave or reset reconciliation. - if err := publishAndClearAgentUpgradeSignal(ctx, log, directClient, upgrades); err != nil { - // Retain the signal so a later daemon start can retry publication. - log.Warn("failed to publish AgentUpgrade startup result", "error", err) - } + go func() { + // Keep host-mutating reconciliation gated until the candidate has reached + // cache readiness and remained alive for a bounded stability interval. + if !mgr.GetCache().WaitForCacheSync(ctx) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(agentUpgradeStartupStability): + } + publishErr := publishAndClearAgentUpgradeSignal(ctx, log, directClient, upgrades) + if publishErr != nil { + // Retain the signal so a later daemon start can retry publication. + log.Warn("failed to publish AgentUpgrade startup result", "error", publishErr) + } else if pending, readErr := upgrades.signals.read(); readErr == nil && pending != nil { + // Recovery scheduled a restart. Keep reconciliation gated until the + // last-good process starts and consumes the retained signal. + return + } + gate.open() + }() err = mgr.Start(ctx) repaves.log.Info("daemon shutting down") diff --git a/pkg/daemon/lifecycle.go b/pkg/daemon/lifecycle.go index 8c5282d4..1a35ba52 100644 --- a/pkg/daemon/lifecycle.go +++ b/pkg/daemon/lifecycle.go @@ -102,10 +102,12 @@ func desiredAgentServiceAssets(binaryPaths agentUpgradePaths, systemdDir, recove } { recoveryContent = bytes.ReplaceAll(recoveryContent, []byte(oldPath), []byte(newPath)) } + // Publish dependencies before the main unit that references OnFailure, so an + // interrupted update never leaves systemd pointing at missing recovery assets. return []agentServiceAsset{ - {path: filepath.Join(systemdDir, ServiceUnitName), content: serviceContent, mode: 0o644}, - {path: filepath.Join(systemdDir, recoveryServiceUnitName), content: recoveryServiceContent, mode: 0o644}, {path: recoveryScript, content: recoveryContent, mode: 0o750}, + {path: filepath.Join(systemdDir, recoveryServiceUnitName), content: recoveryServiceContent, mode: 0o644}, + {path: filepath.Join(systemdDir, ServiceUnitName), content: serviceContent, mode: 0o644}, } } diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index 8539c8de..c6638bae 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -225,8 +225,11 @@ func (h *machineOperationHandlers) beginAgentUpgradeRecovery( // loop can retry recovery without waiting for another reconciliation. h.log.Error("failed to restart daemon for AgentUpgrade recovery", "operation", op.Name, "error", restartErr) } - if recordErr != nil && restartErr != nil { - return ctrl.Result{}, errors.Join(recordErr, restartErr) + if recordErr != nil || restartErr != nil { + return ctrl.Result{}, errors.Join( + wrapOptionalError("record AgentUpgrade recovery failure", recordErr), + wrapOptionalError("restart daemon for AgentUpgrade recovery", restartErr), + ) } return ctrl.Result{}, nil } diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index 6a70c1df..f950b059 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -449,6 +449,25 @@ func TestMachineOperationHandlersAgentUpgradeAbortFailureStartsRecovery(t *testi } } +func TestMachineOperationHandlersAgentUpgradeRecoveryRestartFailureRequeues(t *testing.T) { + t.Parallel() + + upgrader := &fakeAgentUpgradeExecutor{ + stageErr: errors.New("stage failed"), + abortErr: errors.New("rollback failed"), + restartErr: errors.New("restart failed"), + } + target := &machineOperationHandlers{log: slog.Default(), operator: &fakeNodeOperator{}, agentUpgrade: upgrader} + op := daemon.MachineOperation{Name: "upgrade-1", Parameters: map[string]string{ + agentUpgradeDownloadURLParameter: "https://example.com/agent.tar.gz", + }} + + _, err := target.reconcileAgentUpgrade(t.Context(), &fakeMachineOperationStore{}, op) + if err == nil || !strings.Contains(err.Error(), "restart daemon for AgentUpgrade recovery") { + t.Fatalf("reconcileAgentUpgrade error = %v, want recovery restart error", err) + } +} + func TestMachineOperationHandlersAgentReset(t *testing.T) { t.Parallel() diff --git a/pkg/daemon/startup_gate.go b/pkg/daemon/startup_gate.go new file mode 100644 index 00000000..20b084e5 --- /dev/null +++ b/pkg/daemon/startup_gate.go @@ -0,0 +1,65 @@ +package daemon + +import ( + "context" + "sync" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + + agentdaemon "github.com/Azure/unbounded/pkg/agent/daemon" +) + +type startupGate struct { + ready chan struct{} + once sync.Once +} + +func newStartupGate() *startupGate { + return &startupGate{ready: make(chan struct{})} +} + +func (g *startupGate) open() { + g.once.Do(func() { close(g.ready) }) +} + +func (g *startupGate) wait(ctx context.Context) error { + select { + case <-g.ready: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +type gatedMachineOperationReconciler struct { + delegate agentdaemon.MachineOperationRequestReconciler + gate *startupGate +} + +func (r gatedMachineOperationReconciler) SetupController(b *builder.TypedBuilder[agentdaemon.Request]) *builder.TypedBuilder[agentdaemon.Request] { + return r.delegate.SetupController(b) +} + +func (r gatedMachineOperationReconciler) ReconcileMachineOperation(ctx context.Context, name string) (ctrl.Result, error) { + if err := r.gate.wait(ctx); err != nil { + return ctrl.Result{}, err + } + return r.delegate.ReconcileMachineOperation(ctx, name) +} + +type gatedRepaveReconciler struct { + delegate agentdaemon.RepaveReconciler + gate *startupGate +} + +func (r gatedRepaveReconciler) SetupController(b *builder.TypedBuilder[agentdaemon.Request]) *builder.TypedBuilder[agentdaemon.Request] { + return r.delegate.SetupController(b) +} + +func (r gatedRepaveReconciler) ReconcileRepave(ctx context.Context, source string) (ctrl.Result, error) { + if err := r.gate.wait(ctx); err != nil { + return ctrl.Result{}, err + } + return r.delegate.ReconcileRepave(ctx, source) +} diff --git a/pkg/daemon/startup_gate_test.go b/pkg/daemon/startup_gate_test.go new file mode 100644 index 00000000..72e226e0 --- /dev/null +++ b/pkg/daemon/startup_gate_test.go @@ -0,0 +1,41 @@ +package daemon + +import ( + "context" + "errors" + "testing" +) + +func TestStartupGate(t *testing.T) { + t.Parallel() + + t.Run("blocks until opened", func(t *testing.T) { + t.Parallel() + gate := newStartupGate() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { done <- gate.wait(ctx) }() + + select { + case err := <-done: + t.Fatalf("wait returned before open: %v", err) + default: + } + gate.open() + if err := <-done; err != nil { + t.Fatalf("wait after open: %v", err) + } + gate.open() // idempotent + }) + + t.Run("honors cancellation", func(t *testing.T) { + t.Parallel() + gate := newStartupGate() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := gate.wait(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("wait error = %v, want context.Canceled", err) + } + }) +} From e110e34e869ecc2da060f118e1df3198f3f83663 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 05:17:18 +0000 Subject: [PATCH 41/45] fix: retry unresolved upgrade handoffs --- pkg/daemon/agent_upgrade.go | 17 ++++++++ pkg/daemon/daemon.go | 32 +++++++++++---- pkg/daemon/host_agent_activation.go | 41 ++++++++++++++----- pkg/daemon/host_agent_activation_test.go | 21 ++++++++++ pkg/daemon/machineoperation_reconciler.go | 7 ++-- .../machineoperation_reconciler_test.go | 20 +++++++++ 6 files changed, 114 insertions(+), 24 deletions(-) diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index c37c68d4..62fff62b 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -172,6 +172,7 @@ func (s agentUpgradeSignalStore) clear() error { type agentUpgradeExecutor interface { Acquire() (io.Closer, error) RecordPending(context.Context, string) error + RetryRecovery(context.Context) error RecordFailure(string) error Stage(context.Context, agentUpgradeRequest) error Abort(context.Context) error @@ -249,6 +250,22 @@ func (e *hostAgentUpgradeExecutor) RecordPending(ctx context.Context, operationN return nil } +func (e *hostAgentUpgradeExecutor) RetryRecovery(ctx context.Context) error { + signal, err := e.signals.read() + if err != nil { + return err + } + if signal == nil || !signal.RecoveryRequired { + return nil + } + cleanupCtx, cancel := agentUpgradeCleanupContext(ctx) + defer cancel() + if err := e.Restart(cleanupCtx); err != nil { + return fmt.Errorf("retry AgentUpgrade recovery restart: %w", err) + } + return nil +} + func (e *hostAgentUpgradeExecutor) RecordFailure(message string) error { return e.signals.recordFailure(message) } diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index a3c8311b..619dfb44 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -26,6 +26,7 @@ import ( const ( agentUpgradeStartupStability = 3 * time.Second + agentUpgradePublishRetry = 10 * time.Second daemonCredentialDir = "daemon-credentials" //nolint:gosec // Directory name, not a credential. daemonCredentialGroup = "aks-flex-node-daemons" //nolint:gosec // Kubernetes group name, not a credential. ) @@ -127,16 +128,29 @@ func Run(ctx context.Context, cfg *config.Config, log *slog.Logger) error { return case <-time.After(agentUpgradeStartupStability): } - publishErr := publishAndClearAgentUpgradeSignal(ctx, log, directClient, upgrades) - if publishErr != nil { - // Retain the signal so a later daemon start can retry publication. - log.Warn("failed to publish AgentUpgrade startup result", "error", publishErr) - } else if pending, readErr := upgrades.signals.read(); readErr == nil && pending != nil { - // Recovery scheduled a restart. Keep reconciliation gated until the - // last-good process starts and consumes the retained signal. - return + for { + publishErr := publishAndClearAgentUpgradeSignal(ctx, log, directClient, upgrades) + pending, readErr := upgrades.signals.read() + switch { + case publishErr != nil: + log.Warn("failed to publish AgentUpgrade startup result; will retry", "error", publishErr) + case readErr != nil: + log.Warn("failed to confirm AgentUpgrade startup signal; will retry", "error", readErr) + case pending != nil: + // Recovery scheduled a restart. Keep reconciliation gated until the + // last-good process starts and consumes the retained signal. + return + default: + gate.open() + return + } + + select { + case <-ctx.Done(): + return + case <-time.After(agentUpgradePublishRetry): + } } - gate.open() }() err = mgr.Start(ctx) diff --git a/pkg/daemon/host_agent_activation.go b/pkg/daemon/host_agent_activation.go index 8c9c3a23..5559c7de 100644 --- a/pkg/daemon/host_agent_activation.go +++ b/pkg/daemon/host_agent_activation.go @@ -60,7 +60,7 @@ type flexDaemonActivationService struct { state stateStore systemdDir string recoveryScript string - isServiceActive func(context.Context, *slog.Logger, string) bool + inspectService func(context.Context, *slog.Logger, string) (bool, error) serviceWasActive bool } @@ -74,21 +74,25 @@ func newFlexDaemonActivationService(log *slog.Logger) (*flexDaemonActivationServ } paths := defaultAgentUpgradePaths() return &flexDaemonActivationService{ - log: log, - paths: paths, - state: state, - systemdDir: systemdSystemDir, - recoveryScript: recoveryScriptPath, - isServiceActive: utilexec.IsServiceActive, + log: log, + paths: paths, + state: state, + systemdDir: systemdSystemDir, + recoveryScript: recoveryScriptPath, + inspectService: inspectAgentServiceActive, }, paths, nil } func (s *flexDaemonActivationService) Preflight(ctx context.Context, currentBinaryPath string) (agentbinary.ServicePlan, error) { - isActive := s.isServiceActive - if isActive == nil { - isActive = utilexec.IsServiceActive + inspectService := s.inspectService + if inspectService == nil { + inspectService = inspectAgentServiceActive } - s.serviceWasActive = isActive(ctx, s.log, ServiceUnitName) + serviceWasActive, err := inspectService(ctx, s.log, ServiceUnitName) + if err != nil { + return agentbinary.ServicePlan{}, fmt.Errorf("inspect agent service state: %w", err) + } + s.serviceWasActive = serviceWasActive if _, err := os.Stat(s.paths.SignalPath); err == nil { return agentbinary.ServicePlan{}, fmt.Errorf("AgentUpgrade MachineOperation signal exists at %s", s.paths.SignalPath) } else if !errors.Is(err, os.ErrNotExist) { @@ -109,6 +113,21 @@ func (s *flexDaemonActivationService) Preflight(ctx context.Context, currentBina return agentbinary.ServicePlan{Description: "AKS Flex Node agent systemd assets are current"}, nil } +func inspectAgentServiceActive(ctx context.Context, log *slog.Logger, service string) (bool, error) { + state, err := utilexec.OutputCmdAt(ctx, log, slog.LevelDebug, "systemctl", "show", "--property=ActiveState", "--value", service) + if err != nil { + return false, err + } + switch strings.TrimSpace(state) { + case "inactive": + return false, nil + case "active", "activating", "reloading", "deactivating", "failed": + return true, nil + default: + return false, fmt.Errorf("unexpected ActiveState %q for %s", state, service) + } +} + func (s *flexDaemonActivationService) Prepare(_ context.Context, currentBinaryPath string) error { return writeAgentServiceAssets(s.paths, s.systemdDir, s.recoveryScript, currentBinaryPath) } diff --git a/pkg/daemon/host_agent_activation_test.go b/pkg/daemon/host_agent_activation_test.go index 3f7d044c..aaf694d7 100644 --- a/pkg/daemon/host_agent_activation_test.go +++ b/pkg/daemon/host_agent_activation_test.go @@ -1,6 +1,7 @@ package daemon import ( + "context" "errors" "log/slog" "os" @@ -19,6 +20,9 @@ func TestFlexDaemonActivationPreflightUsesFlexAssetsWithoutMutation(t *testing.T paths: paths, systemdDir: systemdDir, recoveryScript: recoveryScript, + inspectService: func(context.Context, *slog.Logger, string) (bool, error) { + return false, nil + }, } plan, err := service.Preflight(t.Context(), paths.CurrentPath) if err != nil { @@ -53,6 +57,20 @@ func TestFlexDaemonActivationLeavesInactiveResetHostStopped(t *testing.T) { } } +func TestFlexDaemonActivationPreflightRejectsUnknownServiceState(t *testing.T) { + t.Parallel() + + service := &flexDaemonActivationService{ + log: slog.Default(), + inspectService: func(context.Context, *slog.Logger, string) (bool, error) { + return false, errors.New("systemd unavailable") + }, + } + if _, err := service.Preflight(t.Context(), "/unused/current"); err == nil { + t.Fatal("Preflight accepted an unknown service state") + } +} + func TestFlexDaemonActivationPreflightRejectsMachineOperationSignal(t *testing.T) { t.Parallel() @@ -68,6 +86,9 @@ func TestFlexDaemonActivationPreflightRejectsMachineOperationSignal(t *testing.T paths: paths, systemdDir: t.TempDir(), recoveryScript: filepath.Join(t.TempDir(), "recovery.sh"), + inspectService: func(context.Context, *slog.Logger, string) (bool, error) { + return false, nil + }, } if _, err := service.Preflight(t.Context(), paths.CurrentPath); err == nil { t.Fatal("Preflight accepted a pending MachineOperation signal") diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index c6638bae..5d367fea 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -170,10 +170,9 @@ func (h *machineOperationHandlers) reconcileAgentUpgrade( // must exist before the status can become non-reconcilable. if err := h.agentUpgrade.RecordPending(ctx, op.Name); err != nil { if errors.Is(err, errAgentUpgradeAlreadyPending) { - // An InProgress status event can already be queued before the delayed - // daemon restart. The durable signal proves this operation was staged - // by an earlier reconciliation. - return ctrl.Result{}, nil + // Retry a previously failed recovery handoff. An ordinary duplicate + // remains a no-op while the delayed daemon restart is pending. + return ctrl.Result{}, h.agentUpgrade.RetryRecovery(ctx) } return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", err.Error()) } diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index f950b059..d682b1bc 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -466,6 +466,16 @@ func TestMachineOperationHandlersAgentUpgradeRecoveryRestartFailureRequeues(t *t if err == nil || !strings.Contains(err.Error(), "restart daemon for AgentUpgrade recovery") { t.Fatalf("reconcileAgentUpgrade error = %v, want recovery restart error", err) } + + upgrader.pendingErr = errAgentUpgradeAlreadyPending + upgrader.restartErr = nil + upgrader.restarted = false + if _, err := target.reconcileAgentUpgrade(t.Context(), &fakeMachineOperationStore{}, op); err != nil { + t.Fatalf("reconcileAgentUpgrade retry: %v", err) + } + if !upgrader.restarted { + t.Fatal("requeued recovery did not retry the daemon restart") + } } func TestMachineOperationHandlersAgentReset(t *testing.T) { @@ -538,6 +548,7 @@ type fakeAgentUpgradeExecutor struct { aborted bool restarted bool failure string + recovering bool acquireErr error pendingErr error stageErr error @@ -557,8 +568,17 @@ func (f *fakeAgentUpgradeExecutor) RecordPending(context.Context, string) error return f.pendingErr } +func (f *fakeAgentUpgradeExecutor) RetryRecovery(context.Context) error { + if !f.recovering { + return nil + } + f.restarted = true + return f.restartErr +} + func (f *fakeAgentUpgradeExecutor) RecordFailure(message string) error { f.failure = message + f.recovering = true return nil } From 11e42cad9a6bce7e6e052449043fcb67f582dd05 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 05:42:14 +0000 Subject: [PATCH 42/45] fix: hold reconciliation through daemon handoff --- docs/usages/operations.md | 2 +- pkg/daemon/agent_upgrade.go | 12 ++++++++++++ pkg/daemon/machineoperation_reconciler.go | 14 +++++++++++++- pkg/daemon/machineoperation_reconciler_test.go | 11 +++++++++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/usages/operations.md b/docs/usages/operations.md index bfddcdab..bd2bbc9b 100644 --- a/docs/usages/operations.md +++ b/docs/usages/operations.md @@ -77,7 +77,7 @@ kubectl get machineoperation upgrade-agent-worker-01 -w A host provisioning system that has already authenticated and staged a candidate can activate it directly without creating an Unbounded `MachineOperation`: ```bash -/var/tmp/aks-flex-node-candidate agent-upgrade --preflight +sudo /var/tmp/aks-flex-node-candidate agent-upgrade --preflight sudo /var/tmp/aks-flex-node-candidate agent-upgrade ``` diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 62fff62b..60c17fb5 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -177,6 +177,7 @@ type agentUpgradeExecutor interface { Stage(context.Context, agentUpgradeRequest) error Abort(context.Context) error Restart(context.Context) error + WaitForRestart(context.Context) error } type agentUpgradeStateLoader interface { @@ -346,6 +347,17 @@ func (e *hostAgentUpgradeExecutor) rollback(ctx context.Context) error { return rollbackAgentUpgradeFiles(e.paths, signal) } +func (e *hostAgentUpgradeExecutor) WaitForRestart(ctx context.Context) error { + timer := time.NewTimer(10 * time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + return nil + case <-timer.C: + return fmt.Errorf("timed out waiting for scheduled daemon restart") + } +} + func (e *hostAgentUpgradeExecutor) Restart(ctx context.Context) error { if err := ctx.Err(); err != nil { return err diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index 5d367fea..bc5f5ba3 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -81,7 +81,11 @@ func machineOperationReconciler( return nil, fmt.Errorf("AKS machine name is empty") } - handlers := &machineOperationHandlers{log: opts.Log, operator: opts.Operator, agentUpgrade: opts.AgentUpgrade} + handlers := &machineOperationHandlers{ + log: opts.Log, + operator: opts.Operator, + agentUpgrade: opts.AgentUpgrade, + } reconciler, err := daemon.NewMachinaMachineOperationReconciler( opts.Client, opts.NodeName, @@ -197,6 +201,14 @@ func (h *machineOperationHandlers) reconcileAgentUpgrade( } return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", "failed to restart upgraded agent daemon") } + // Keep the single-worker controller occupied until systemd stops this + // process. That closes the delayed-restart window to queued host mutations. + if err := h.agentUpgrade.WaitForRestart(ctx); err != nil { + if abortErr := h.agentUpgrade.Abort(ctx); abortErr != nil { + return h.beginAgentUpgradeRecovery(ctx, op, err, abortErr) + } + return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", err.Error()) + } // The restarted daemon publishes success after proving the candidate can // initialize its Kubernetes client and controller. return ctrl.Result{}, nil diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index d682b1bc..53749a82 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -310,8 +310,8 @@ func TestMachineOperationHandlersAgentUpgrade(t *testing.T) { if _, err := target.reconcileAgentUpgrade(t.Context(), store, op); err != nil { t.Fatalf("reconcileAgentUpgrade: %v", err) } - if !store.inProgress || !upgrader.pending || !upgrader.staged || !upgrader.restarted { - t.Fatalf("upgrade calls = inProgress:%v pending:%v staged:%v restarted:%v", store.inProgress, upgrader.pending, upgrader.staged, upgrader.restarted) + if !store.inProgress || !upgrader.pending || !upgrader.staged || !upgrader.restarted || !upgrader.waited { + t.Fatalf("upgrade calls = inProgress:%v pending:%v staged:%v restarted:%v waited:%v", store.inProgress, upgrader.pending, upgrader.staged, upgrader.restarted, upgrader.waited) } if store.result.Phase != "" { t.Fatalf("phase = %s, want non-terminal until daemon restart", store.result.Phase) @@ -554,6 +554,8 @@ type fakeAgentUpgradeExecutor struct { stageErr error abortErr error restartErr error + waitErr error + waited bool } func (f *fakeAgentUpgradeExecutor) Acquire() (io.Closer, error) { @@ -597,6 +599,11 @@ func (f *fakeAgentUpgradeExecutor) Restart(context.Context) error { return f.restartErr } +func (f *fakeAgentUpgradeExecutor) WaitForRestart(context.Context) error { + f.waited = true + return f.waitErr +} + var _ agentUpgradeExecutor = (*fakeAgentUpgradeExecutor)(nil) type fakeMachineOperationStore struct { From 176d945bd57fcf535a23e8b53968046e99f6c170 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 05:44:06 +0000 Subject: [PATCH 43/45] fix: preserve recovery handoff serialization --- pkg/daemon/agent_upgrade.go | 4 ++-- pkg/daemon/agent_upgrade_test.go | 4 ++-- pkg/daemon/machineoperation_reconciler.go | 4 +--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 60c17fb5..dceb4061 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -264,7 +264,7 @@ func (e *hostAgentUpgradeExecutor) RetryRecovery(ctx context.Context) error { if err := e.Restart(cleanupCtx); err != nil { return fmt.Errorf("retry AgentUpgrade recovery restart: %w", err) } - return nil + return e.WaitForRestart(ctx) } func (e *hostAgentUpgradeExecutor) RecordFailure(message string) error { @@ -560,7 +560,7 @@ func rollbackAgentUpgradeFiles(paths agentUpgradePaths, signal *agentUpgradeSign if err != nil { return err } - if !signal.SwitchCommitted && !candidateActive { + if !signal.SwitchCommitted && !signal.RecoveryRequired && !candidateActive { return nil } lastGood, err := resolvedExecutable(paths.LastGoodPath) diff --git a/pkg/daemon/agent_upgrade_test.go b/pkg/daemon/agent_upgrade_test.go index d7de89de..89596a53 100644 --- a/pkg/daemon/agent_upgrade_test.go +++ b/pkg/daemon/agent_upgrade_test.go @@ -413,8 +413,8 @@ func TestRollbackAgentUpgradeFilesRetriesAfterHostLinkWasRestored(t *testing.T) } err := rollbackAgentUpgradeFiles(paths, &agentUpgradeSignal{ - CandidatePath: paths.BluePath, - SwitchCommitted: true, + CandidatePath: paths.BluePath, + RecoveryRequired: true, }) if err == nil || !strings.Contains(err.Error(), "resolve last-good") { t.Fatalf("rollbackAgentUpgradeFiles error = %v, want retry to resolve last-good", err) diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index bc5f5ba3..c197bb59 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -232,8 +232,6 @@ func (h *machineOperationHandlers) beginAgentUpgradeRecovery( defer cancel() restartErr := h.agentUpgrade.Restart(cleanupCtx) if restartErr != nil { - // If the failure annotation succeeded, the initiating daemon's signal - // loop can retry recovery without waiting for another reconciliation. h.log.Error("failed to restart daemon for AgentUpgrade recovery", "operation", op.Name, "error", restartErr) } if recordErr != nil || restartErr != nil { @@ -242,7 +240,7 @@ func (h *machineOperationHandlers) beginAgentUpgradeRecovery( wrapOptionalError("restart daemon for AgentUpgrade recovery", restartErr), ) } - return ctrl.Result{}, nil + return ctrl.Result{}, h.agentUpgrade.WaitForRestart(ctx) } func (h *machineOperationHandlers) reconcileAgentReset( From 09befda306f1739f5d07188a7018a2a191b46c71 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Tue, 11 Aug 2026 05:48:23 +0000 Subject: [PATCH 44/45] test: cover successful upgrade publication --- pkg/daemon/agent_upgrade.go | 16 +++++++-- pkg/daemon/agent_upgrade_test.go | 54 +++++++++++++++++++++++++++++ pkg/daemon/host_agent_activation.go | 9 +++++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index dceb4061..739a70f6 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -192,6 +192,7 @@ type hostAgentUpgradeExecutor struct { runSystemdRun func(context.Context, ...string) error finishMachineOperation func(context.Context, client.Client, agentdaemon.MachineOperation, agentdaemon.MachineOperationResult[int64]) error runningExecutable func() (string, error) + nspawnBinaryPath func(string) string instanceID string } @@ -211,6 +212,7 @@ func newHostAgentUpgradeExecutor(log *slog.Logger, state agentUpgradeStateLoader }, finishMachineOperation: agentdaemon.FinishMachineOperation, runningExecutable: runningAgentExecutable, + nspawnBinaryPath: activeNspawnAgentBinaryPath, instanceID: instanceID, }, nil } @@ -427,7 +429,11 @@ func publishAndClearAgentUpgradeSignal(ctx context.Context, log *slog.Logger, c } if signal.Failure == "" { - if validationErr := validateStartedAgentUpgrade(paths, signal); validationErr != nil { + nspawnBinaryPath := executor.nspawnBinaryPath + if nspawnBinaryPath == nil { + nspawnBinaryPath = activeNspawnAgentBinaryPath + } + if validationErr := validateStartedAgentUpgrade(paths, signal, nspawnBinaryPath); validationErr != nil { signal.Failure = validationErr.Error() candidateActive, activeErr := agentUpgradeCandidateIsActive(paths, signal) if activeErr != nil { @@ -516,7 +522,7 @@ func wrapOptionalError(context string, err error) error { return fmt.Errorf("%s: %w", context, err) } -func validateStartedAgentUpgrade(paths agentUpgradePaths, signal *agentUpgradeSignal) error { +func validateStartedAgentUpgrade(paths agentUpgradePaths, signal *agentUpgradeSignal, nspawnBinaryPath func(string) string) error { if signal.CandidatePath != paths.BluePath && signal.CandidatePath != paths.GreenPath { return fmt.Errorf("AgentUpgrade was interrupted before selecting a candidate slot") } @@ -530,7 +536,7 @@ func validateStartedAgentUpgrade(paths agentUpgradePaths, signal *agentUpgradeSi if !validNspawnMachine(signal.ActiveMachine) { return fmt.Errorf("AgentUpgrade has no valid active nspawn machine") } - nspawnPath := filepath.Join("/var/lib/machines", signal.ActiveMachine, "usr", "local", "bin", "aks-flex-node") + nspawnPath := nspawnBinaryPath(signal.ActiveMachine) equal, err := filesHaveEqualSHA256(current, nspawnPath) if err != nil { return fmt.Errorf("verify synchronized nspawn agent binary: %w", err) @@ -552,6 +558,10 @@ func agentUpgradeCandidateIsActive(paths agentUpgradePaths, signal *agentUpgrade return current == signal.CandidatePath, nil } +func activeNspawnAgentBinaryPath(machine string) string { + return filepath.Join("/var/lib/machines", machine, "usr", "local", "bin", "aks-flex-node") +} + func rollbackAgentUpgradeFiles(paths agentUpgradePaths, signal *agentUpgradeSignal) error { if signal == nil { return nil diff --git a/pkg/daemon/agent_upgrade_test.go b/pkg/daemon/agent_upgrade_test.go index 89596a53..37a0f679 100644 --- a/pkg/daemon/agent_upgrade_test.go +++ b/pkg/daemon/agent_upgrade_test.go @@ -354,6 +354,60 @@ func TestPublishAgentUpgradeFailureRestartsIntoLastGoodBeforeClearingSignal(t *t } } +func TestPublishAgentUpgradeSuccessCompletesAndClearsSignal(t *testing.T) { + t.Parallel() + + paths := testAgentUpgradePaths(t) + if err := os.MkdirAll(filepath.Dir(paths.BluePath), 0o750); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + if err := os.WriteFile(paths.BluePath, []byte("candidate"), 0o755); err != nil { + t.Fatalf("WriteFile candidate: %v", err) + } + if err := os.Symlink(paths.BluePath, paths.CurrentPath); err != nil { + t.Fatalf("Symlink current: %v", err) + } + nspawnBinary := filepath.Join(t.TempDir(), "nspawn-agent") + if err := os.WriteFile(nspawnBinary, []byte("candidate"), 0o755); err != nil { + t.Fatalf("WriteFile nspawn: %v", err) + } + signals := agentUpgradeSignalStore{path: paths.SignalPath} + if err := signals.write(agentUpgradeSignal{ + OperationName: "operation-1", + ActiveMachine: "kube1", + CandidatePath: paths.BluePath, + InitiatingDaemonInstance: "previous-instance", + SwitchCommitted: true, + }); err != nil { + t.Fatalf("write signal: %v", err) + } + finished := 0 + executor := &hostAgentUpgradeExecutor{ + paths: paths, + signals: signals, + instanceID: "restarted-instance", + nspawnBinaryPath: func(string) string { + return nspawnBinary + }, + finishMachineOperation: func(_ context.Context, _ client.Client, _ agentdaemon.MachineOperation, result agentdaemon.MachineOperationResult[int64]) error { + finished++ + if result.Phase != machinav1alpha3.OperationPhaseComplete { + t.Fatalf("phase = %s, want Complete", result.Phase) + } + return nil + }, + } + if err := publishAndClearAgentUpgradeSignal(t.Context(), slog.Default(), nil, executor); err != nil { + t.Fatalf("publishAndClearAgentUpgradeSignal: %v", err) + } + if finished != 1 { + t.Fatalf("finish calls = %d, want 1", finished) + } + if signal, err := signals.read(); err != nil || signal != nil { + t.Fatalf("signal after success = %#v, %v", signal, err) + } +} + func TestPublishAgentUpgradeSignalIgnoresInitiatingProcess(t *testing.T) { t.Parallel() diff --git a/pkg/daemon/host_agent_activation.go b/pkg/daemon/host_agent_activation.go index 5559c7de..db3f3568 100644 --- a/pkg/daemon/host_agent_activation.go +++ b/pkg/daemon/host_agent_activation.go @@ -114,6 +114,15 @@ func (s *flexDaemonActivationService) Preflight(ctx context.Context, currentBina } func inspectAgentServiceActive(ctx context.Context, log *slog.Logger, service string) (bool, error) { + loadState, err := utilexec.OutputCmdAt(ctx, log, slog.LevelDebug, "systemctl", "show", "--property=LoadState", "--value", service) + if err != nil { + return false, err + } + if strings.TrimSpace(loadState) == "not-found" { + // Reset removes the unit while intentionally retaining the managed binary + // layout for a later rejoin. + return false, nil + } state, err := utilexec.OutputCmdAt(ctx, log, slog.LevelDebug, "systemctl", "show", "--property=ActiveState", "--value", service) if err != nil { return false, err From f977bbdc9522183d6c9ddfc0c954015f52609097 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Wed, 12 Aug 2026 03:44:56 +0000 Subject: [PATCH 45/45] fix: let restarted daemon own terminal status --- pkg/daemon/agent_upgrade.go | 14 +------------- pkg/daemon/machineoperation_reconciler.go | 15 ++++----------- pkg/daemon/machineoperation_reconciler_test.go | 11 ++--------- 3 files changed, 7 insertions(+), 33 deletions(-) diff --git a/pkg/daemon/agent_upgrade.go b/pkg/daemon/agent_upgrade.go index 739a70f6..a18d7617 100644 --- a/pkg/daemon/agent_upgrade.go +++ b/pkg/daemon/agent_upgrade.go @@ -177,7 +177,6 @@ type agentUpgradeExecutor interface { Stage(context.Context, agentUpgradeRequest) error Abort(context.Context) error Restart(context.Context) error - WaitForRestart(context.Context) error } type agentUpgradeStateLoader interface { @@ -266,7 +265,7 @@ func (e *hostAgentUpgradeExecutor) RetryRecovery(ctx context.Context) error { if err := e.Restart(cleanupCtx); err != nil { return fmt.Errorf("retry AgentUpgrade recovery restart: %w", err) } - return e.WaitForRestart(ctx) + return nil } func (e *hostAgentUpgradeExecutor) RecordFailure(message string) error { @@ -349,17 +348,6 @@ func (e *hostAgentUpgradeExecutor) rollback(ctx context.Context) error { return rollbackAgentUpgradeFiles(e.paths, signal) } -func (e *hostAgentUpgradeExecutor) WaitForRestart(ctx context.Context) error { - timer := time.NewTimer(10 * time.Second) - defer timer.Stop() - select { - case <-ctx.Done(): - return nil - case <-timer.C: - return fmt.Errorf("timed out waiting for scheduled daemon restart") - } -} - func (e *hostAgentUpgradeExecutor) Restart(ctx context.Context) error { if err := ctx.Err(); err != nil { return err diff --git a/pkg/daemon/machineoperation_reconciler.go b/pkg/daemon/machineoperation_reconciler.go index c197bb59..564c63c8 100644 --- a/pkg/daemon/machineoperation_reconciler.go +++ b/pkg/daemon/machineoperation_reconciler.go @@ -201,16 +201,9 @@ func (h *machineOperationHandlers) reconcileAgentUpgrade( } return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", "failed to restart upgraded agent daemon") } - // Keep the single-worker controller occupied until systemd stops this - // process. That closes the delayed-restart window to queued host mutations. - if err := h.agentUpgrade.WaitForRestart(ctx); err != nil { - if abortErr := h.agentUpgrade.Abort(ctx); abortErr != nil { - return h.beginAgentUpgradeRecovery(ctx, op, err, abortErr) - } - return h.finishFailedMachineOperation(ctx, store, op, "ExecutionFailed", err.Error()) - } - // The restarted daemon publishes success after proving the candidate can - // initialize its Kubernetes client and controller. + // Restart scheduling is the old daemon's final responsibility. Keep the + // operation InProgress and let the restarted or recovery daemon publish the + // only terminal result. return ctrl.Result{}, nil } @@ -240,7 +233,7 @@ func (h *machineOperationHandlers) beginAgentUpgradeRecovery( wrapOptionalError("restart daemon for AgentUpgrade recovery", restartErr), ) } - return ctrl.Result{}, h.agentUpgrade.WaitForRestart(ctx) + return ctrl.Result{}, nil } func (h *machineOperationHandlers) reconcileAgentReset( diff --git a/pkg/daemon/machineoperation_reconciler_test.go b/pkg/daemon/machineoperation_reconciler_test.go index 53749a82..d682b1bc 100644 --- a/pkg/daemon/machineoperation_reconciler_test.go +++ b/pkg/daemon/machineoperation_reconciler_test.go @@ -310,8 +310,8 @@ func TestMachineOperationHandlersAgentUpgrade(t *testing.T) { if _, err := target.reconcileAgentUpgrade(t.Context(), store, op); err != nil { t.Fatalf("reconcileAgentUpgrade: %v", err) } - if !store.inProgress || !upgrader.pending || !upgrader.staged || !upgrader.restarted || !upgrader.waited { - t.Fatalf("upgrade calls = inProgress:%v pending:%v staged:%v restarted:%v waited:%v", store.inProgress, upgrader.pending, upgrader.staged, upgrader.restarted, upgrader.waited) + if !store.inProgress || !upgrader.pending || !upgrader.staged || !upgrader.restarted { + t.Fatalf("upgrade calls = inProgress:%v pending:%v staged:%v restarted:%v", store.inProgress, upgrader.pending, upgrader.staged, upgrader.restarted) } if store.result.Phase != "" { t.Fatalf("phase = %s, want non-terminal until daemon restart", store.result.Phase) @@ -554,8 +554,6 @@ type fakeAgentUpgradeExecutor struct { stageErr error abortErr error restartErr error - waitErr error - waited bool } func (f *fakeAgentUpgradeExecutor) Acquire() (io.Closer, error) { @@ -599,11 +597,6 @@ func (f *fakeAgentUpgradeExecutor) Restart(context.Context) error { return f.restartErr } -func (f *fakeAgentUpgradeExecutor) WaitForRestart(context.Context) error { - f.waited = true - return f.waitErr -} - var _ agentUpgradeExecutor = (*fakeAgentUpgradeExecutor)(nil) type fakeMachineOperationStore struct {