diff --git a/devnet/scripts/submit-upgrade-proposal.sh b/devnet/scripts/submit-upgrade-proposal.sh index 71b7819d..fdc74f43 100755 --- a/devnet/scripts/submit-upgrade-proposal.sh +++ b/devnet/scripts/submit-upgrade-proposal.sh @@ -19,7 +19,14 @@ LUMERA_SHARED="/tmp/${CHAIN_ID}/shared" KEYRING="test" HOST_PROPOSAL_FILE="${LUMERA_SHARED}/upgrade_${VERSION}.json" CONTAINER_PROPOSAL_FILE="/shared/upgrade_${VERSION}.json" -COMPOSE_FILE="../docker-compose.yml" +# Resolve paths relative to THIS script, not the caller's working directory. +# Previously this was "../docker-compose.yml", which only resolved when the +# script happened to be invoked from devnet/scripts/. Run from the repository +# root (as documented for the rehearsal flow), every `docker compose -f` call +# silently failed, so key/account lookups returned empty and the script +# misreported working accounts as "not present on-chain". +SUBMIT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-${SUBMIT_SCRIPT_DIR}/../docker-compose.yml}" STATUS_DIR="${LUMERA_SHARED}/status" ACCOUNT_REGISTRY_FILE="${STATUS_DIR}/${SERVICE}/accounts.json" diff --git a/devnet/scripts/upgrade-binaries.sh b/devnet/scripts/upgrade-binaries.sh index da64fd86..69c257ff 100755 --- a/devnet/scripts/upgrade-binaries.sh +++ b/devnet/scripts/upgrade-binaries.sh @@ -42,10 +42,23 @@ binary_version() { return 1 fi + # Allow an explicit override. Binaries built locally from a branch (plain + # `go build`, no -ldflags) print an EMPTY version string, so probing them + # can never succeed. That is the normal case when rehearsing an unreleased + # branch, so refusing to proceed would make branch rehearsals impossible. + # Release artifacts still self-report and need no override. + if [[ -n "${DEVNET_BINARY_VERSION:-}" ]]; then + printf '%s\n' "$(normalize_version "${DEVNET_BINARY_VERSION}")" + return 0 + fi + version="$("${binary}" version 2>/dev/null | head -n 1 | tr -d '\r')" version="$(normalize_version "${version}")" if [[ -z "${version}" ]]; then echo "Failed to determine version for binary: ${binary}" >&2 + echo " Binaries built with plain 'go build' report no version." >&2 + echo " Set DEVNET_BINARY_VERSION= to supply it explicitly, e.g." >&2 + echo " DEVNET_BINARY_VERSION=v1.20.2 ./devnet/scripts/upgrade.sh v1.20.2 auto-height devnet/bin-v1.20.2" >&2 return 1 fi printf '%s\n' "${version}" diff --git a/devnet/scripts/upgrade.sh b/devnet/scripts/upgrade.sh index 3034a421..d790bd10 100755 --- a/devnet/scripts/upgrade.sh +++ b/devnet/scripts/upgrade.sh @@ -216,11 +216,23 @@ if ! detect_upgrade_halt; then LIVE_HEIGHT="$(docker compose -f "${COMPOSE_FILE}" exec -T "${SERVICE}" \ lumerad status 2>/dev/null | jq -r '.sync_info.latest_block_height // empty' 2>/dev/null || true)" if [[ "${LIVE_HEIGHT}" =~ ^[0-9]+$ ]]; then - echo "ERROR: chain is still producing blocks (height ${LIVE_HEIGHT}) and has NOT halted for the ${RELEASE_NAME} upgrade." >&2 - echo " The upgrade did not execute — the proposal likely failed to pass (quorum/threshold)." >&2 - echo " Inspect: lumerad query gov proposal ${PROPOSAL_ID}" >&2 - echo " Refusing to swap binaries; running ${RELEASE_NAME} on un-upgraded state would crash all nodes." >&2 - exit 1 + # A node that has stopped for an upgrade KEEPS SERVING RPC at the halt + # height, so a numeric height is not proof the chain is live. Sample twice + # and compare: only an ADVANCING height means blocks are still being made. + # Without this the guard aborted correct upgrades whose halt height had + # just been reached (observed on the two-hop rehearsal at height 207). + sleep "${HALT_CONFIRM_SECS:-12}" + LIVE_HEIGHT_2="$(docker compose -f "${COMPOSE_FILE}" exec -T "${SERVICE}" \ + lumerad status 2>/dev/null | jq -r '.sync_info.latest_block_height // empty' 2>/dev/null || true)" + if [[ "${LIVE_HEIGHT_2}" =~ ^[0-9]+$ ]] && ((LIVE_HEIGHT_2 > LIVE_HEIGHT)); then + echo "ERROR: chain is still producing blocks (${LIVE_HEIGHT} -> ${LIVE_HEIGHT_2}) and has NOT stopped for the ${RELEASE_NAME} upgrade." >&2 + echo " Check the proposal actually PASSED and that its plan height is in the future:" >&2 + echo " lumerad query gov proposal ${PROPOSAL_ID}" >&2 + echo " lumerad query upgrade plan" >&2 + echo " Refusing to swap binaries; running ${RELEASE_NAME} on un-upgraded state would crash all nodes." >&2 + exit 1 + fi + echo "Height static at ${LIVE_HEIGHT} across two samples — chain is stopped at the upgrade boundary; proceeding." fi echo "⚠️ No upgrade-halt marker found and RPC is unreachable; assuming a genuine halt and proceeding." >&2 fi diff --git a/devnet/scripts/vote-all.sh b/devnet/scripts/vote-all.sh index d325aa14..778c2b85 100755 --- a/devnet/scripts/vote-all.sh +++ b/devnet/scripts/vote-all.sh @@ -11,7 +11,13 @@ CHAIN_ID="lumera-devnet-1" KEYRING_BACKEND="test" PROPOSAL_ID="$1" SERVICE_NAME="supernova_validator_1" -COMPOSE_FILE="../docker-compose.yml" +# Resolve relative to THIS script, not the caller's working directory. +# See submit-upgrade-proposal.sh for the full rationale: a caller-relative +# path made every `docker compose -f` call fail when the script was invoked +# from anywhere other than devnet/scripts/, which surfaced as bogus +# "no votes available" / "unable to resolve key" errors. +VOTE_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-${VOTE_SCRIPT_DIR}/../docker-compose.yml}" FEES="5000ulume" # Gas configuration — use a fixed gas amount by default. `--gas auto` simulates # a gov vote at ~57.9k and even with a 1.3x bump lands right at the real usage @@ -113,11 +119,29 @@ ensure_fee_funds() { return 0 fi echo " 💧 $svc voter has ${spendable}ulume spendable (< ${FEE_MIN}); topping up ${FEE_TOPUP} from ${SERVICE_NAME}/${PRIMARY_FUNDING_KEY}" - topup_json=$(docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" \ - lumerad tx bank send "$PRIMARY_FUNDING_KEY" "$addr" "$FEE_TOPUP" \ - --keyring-backend "$KEYRING_BACKEND" --chain-id "$CHAIN_ID" \ - --gas "$GAS_AMOUNT" --gas-prices 0.025ulume -y -o json 2>&1) - topup_rc=$? + # Every voter is topped up from the SAME funding account in a tight loop, so + # back-to-back sends race the account sequence: the second is broadcast before + # the first is committed and fails with "account sequence mismatch, expected N, + # got N-1". The caller then SKIPS that validator's vote entirely — observed + # twice during the Phase 2 rehearsals, each time dropping one of five votes + # (4000000000000 instead of 5000000000000). The proposal still passed there, + # but on a network where one validator carries decisive weight this silently + # changes a governance outcome. Retry on sequence mismatch specifically. + local attempt=1 max_attempts="${TOPUP_MAX_ATTEMPTS:-4}" + while :; do + topup_json=$(docker compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" \ + lumerad tx bank send "$PRIMARY_FUNDING_KEY" "$addr" "$FEE_TOPUP" \ + --keyring-backend "$KEYRING_BACKEND" --chain-id "$CHAIN_ID" \ + --gas "$GAS_AMOUNT" --gas-prices 0.025ulume -y -o json 2>&1) + topup_rc=$? + if echo "$topup_json" | grep -q "account sequence mismatch" && [ "$attempt" -lt "$max_attempts" ]; then + echo " ⏳ $svc: sequence mismatch on top-up (attempt ${attempt}/${max_attempts}); waiting for the previous send to commit" + sleep "${TOPUP_RETRY_SECS:-6}" + attempt=$((attempt + 1)) + continue + fi + break + done if [ "$topup_rc" -ne 0 ]; then echo " ❌ $svc: fee top-up command failed with exit code $topup_rc" >&2 echo "$topup_json" >&2