fix(devnet): resolve upgrade script paths relative to the script, not the caller

devnet/scripts/submit-upgrade-proposal.sh

@@ -19,7 +19,14 @@ LUMERA_SHARED="/tmp/${CHAIN_ID}/shared"
 KEYRING="test"
 HOST_PROPOSAL_FILE="${LUMERA_SHARED}/upgrade_${VERSION}.json"
 CONTAINER_PROPOSAL_FILE="/shared/upgrade_${VERSION}.json"
-COMPOSE_FILE="../docker-compose.yml"
+# Resolve paths relative to THIS script, not the caller's working directory.
+# Previously this was "../docker-compose.yml", which only resolved when the
+# script happened to be invoked from devnet/scripts/. Run from the repository
+# root (as documented for the rehearsal flow), every `docker compose -f` call
+# silently failed, so key/account lookups returned empty and the script
+# misreported working accounts as "not present on-chain".
+SUBMIT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+COMPOSE_FILE="${COMPOSE_FILE:-${SUBMIT_SCRIPT_DIR}/../docker-compose.yml}"
 STATUS_DIR="${LUMERA_SHARED}/status"
 ACCOUNT_REGISTRY_FILE="${STATUS_DIR}/${SERVICE}/accounts.json"
 

devnet/scripts/upgrade-binaries.sh

@@ -42,10 +42,23 @@ binary_version() {
 		return 1
 	fi
 
+	# Allow an explicit override. Binaries built locally from a branch (plain
+	# `go build`, no -ldflags) print an EMPTY version string, so probing them
+	# can never succeed. That is the normal case when rehearsing an unreleased
+	# branch, so refusing to proceed would make branch rehearsals impossible.
+	# Release artifacts still self-report and need no override.
+	if [[ -n "${DEVNET_BINARY_VERSION:-}" ]]; then
+		printf '%s\n' "$(normalize_version "${DEVNET_BINARY_VERSION}")"
+		return 0
+	fi
+
 	version="$("${binary}" version 2>/dev/null | head -n 1 | tr -d '\r')"
 	version="$(normalize_version "${version}")"
 	if [[ -z "${version}" ]]; then
 		echo "Failed to determine version for binary: ${binary}" >&2
+		echo "  Binaries built with plain 'go build' report no version." >&2
+		echo "  Set DEVNET_BINARY_VERSION=<tag> to supply it explicitly, e.g." >&2
+		echo "    DEVNET_BINARY_VERSION=v1.20.2 ./devnet/scripts/upgrade.sh v1.20.2 auto-height devnet/bin-v1.20.2" >&2
 		return 1
 	fi
 	printf '%s\n' "${version}"

devnet/scripts/upgrade.sh

@@ -216,11 +216,23 @@ if ! detect_upgrade_halt; then
 	LIVE_HEIGHT="$(docker compose -f "${COMPOSE_FILE}" exec -T "${SERVICE}" \
 		lumerad status 2>/dev/null | jq -r '.sync_info.latest_block_height // empty' 2>/dev/null || true)"
 	if [[ "${LIVE_HEIGHT}" =~ ^[0-9]+$ ]]; then
-		echo "ERROR: chain is still producing blocks (height ${LIVE_HEIGHT}) and has NOT halted for the ${RELEASE_NAME} upgrade." >&2
-		echo "       The upgrade did not execute — the proposal likely failed to pass (quorum/threshold)." >&2
-		echo "       Inspect: lumerad query gov proposal ${PROPOSAL_ID}" >&2
-		echo "       Refusing to swap binaries; running ${RELEASE_NAME} on un-upgraded state would crash all nodes." >&2
-		exit 1
+		# A node that has stopped for an upgrade KEEPS SERVING RPC at the halt
+		# height, so a numeric height is not proof the chain is live. Sample twice
+		# and compare: only an ADVANCING height means blocks are still being made.
+		# Without this the guard aborted correct upgrades whose halt height had
+		# just been reached (observed on the two-hop rehearsal at height 207).
+		sleep "${HALT_CONFIRM_SECS:-12}"
+		LIVE_HEIGHT_2="$(docker compose -f "${COMPOSE_FILE}" exec -T "${SERVICE}" \
+			lumerad status 2>/dev/null | jq -r '.sync_info.latest_block_height // empty' 2>/dev/null || true)"
+		if [[ "${LIVE_HEIGHT_2}" =~ ^[0-9]+$ ]] && ((LIVE_HEIGHT_2 > LIVE_HEIGHT)); then
+			echo "ERROR: chain is still producing blocks (${LIVE_HEIGHT} -> ${LIVE_HEIGHT_2}) and has NOT stopped for the ${RELEASE_NAME} upgrade." >&2
+			echo "       Check the proposal actually PASSED and that its plan height is in the future:" >&2
+			echo "         lumerad query gov proposal ${PROPOSAL_ID}" >&2
+			echo "         lumerad query upgrade plan" >&2
+			echo "       Refusing to swap binaries; running ${RELEASE_NAME} on un-upgraded state would crash all nodes." >&2
+			exit 1
+		fi
+		echo "Height static at ${LIVE_HEIGHT} across two samples — chain is stopped at the upgrade boundary; proceeding."
 	fi
 	echo "⚠️  No upgrade-halt marker found and RPC is unreachable; assuming a genuine halt and proceeding." >&2
 fi

devnet/scripts/vote-all.sh

@@ -11,7 +11,13 @@ CHAIN_ID="lumera-devnet-1"
 KEYRING_BACKEND="test"
 PROPOSAL_ID="$1"
 SERVICE_NAME="supernova_validator_1"
-COMPOSE_FILE="../docker-compose.yml"
+# Resolve relative to THIS script, not the caller's working directory.
+# See submit-upgrade-proposal.sh for the full rationale: a caller-relative
+# path made every `docker compose -f` call fail when the script was invoked
+# from anywhere other than devnet/scripts/, which surfaced as bogus
+# "no votes available" / "unable to resolve key" errors.
+VOTE_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+COMPOSE_FILE="${COMPOSE_FILE:-${VOTE_SCRIPT_DIR}/../docker-compose.yml}"
 FEES="5000ulume"
 # Gas configuration — use a fixed gas amount by default. `--gas auto` simulates
 # a gov vote at ~57.9k and even with a 1.3x bump lands right at the real usage
@@ -113,11 +119,29 @@ ensure_fee_funds() {
 		return 0
 	fi
 	echo "  💧 $svc voter has ${spendable}ulume spendable (< ${FEE_MIN}); topping up ${FEE_TOPUP} from ${SERVICE_NAME}/${PRIMARY_FUNDING_KEY}"
-	topup_json=$(docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" \
-		lumerad tx bank send "$PRIMARY_FUNDING_KEY" "$addr" "$FEE_TOPUP" \
-		--keyring-backend "$KEYRING_BACKEND" --chain-id "$CHAIN_ID" \
-		--gas "$GAS_AMOUNT" --gas-prices 0.025ulume -y -o json 2>&1)
-	topup_rc=$?
+	# Every voter is topped up from the SAME funding account in a tight loop, so
+	# back-to-back sends race the account sequence: the second is broadcast before
+	# the first is committed and fails with "account sequence mismatch, expected N,
+	# got N-1". The caller then SKIPS that validator's vote entirely — observed
+	# twice during the Phase 2 rehearsals, each time dropping one of five votes
+	# (4000000000000 instead of 5000000000000). The proposal still passed there,
+	# but on a network where one validator carries decisive weight this silently
+	# changes a governance outcome. Retry on sequence mismatch specifically.
+	local attempt=1 max_attempts="${TOPUP_MAX_ATTEMPTS:-4}"
+	while :; do
+		topup_json=$(docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" \
+			lumerad tx bank send "$PRIMARY_FUNDING_KEY" "$addr" "$FEE_TOPUP" \
+			--keyring-backend "$KEYRING_BACKEND" --chain-id "$CHAIN_ID" \
+			--gas "$GAS_AMOUNT" --gas-prices 0.025ulume -y -o json 2>&1)
+		topup_rc=$?
+		if echo "$topup_json" | grep -q "account sequence mismatch" && [ "$attempt" -lt "$max_attempts" ]; then
+			echo "  ⏳ $svc: sequence mismatch on top-up (attempt ${attempt}/${max_attempts}); waiting for the previous send to commit"
+			sleep "${TOPUP_RETRY_SECS:-6}"
+			attempt=$((attempt + 1))
+			continue
+		fi
+		break
+	done
 	if [ "$topup_rc" -ne 0 ]; then
 		echo "  ❌ $svc: fee top-up command failed with exit code $topup_rc" >&2
 		echo "$topup_json" >&2