diff --git a/scripts/measure_e2e.sh b/scripts/measure_e2e.sh new file mode 100644 index 00000000..a419def5 --- /dev/null +++ b/scripts/measure_e2e.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +# SLP transaction end-to-end latency measurement wrapper. +# +# Usage: sh scripts/measure_e2e.sh [SLP_EVAL_OPTION...] [-- MISSION_ARG...] +# +# A preset over scripts/slp_eval.sh that sets the options needed to measure +# tx end-to-end latency. + +SLP_EVAL="$(dirname "$0")/slp_eval.sh" + +NETWORK_SIZE_LIMIT=1000 +PUBNET_DATA_FILE="public-network-data-2026-06-03-trimmed-located.json" +LOADGEN_KEYS_FILE="public-network-data-2026-06-03-loadgenkeys.json" + +# Sets DATA_ROOT and SEPARATOR_GIVEN (whether the user supplied "--" in the +# arguments). Shifts its own copy of the arguments, leaving the caller's "$@" +# intact. +scan_args() { + DATA_ROOT="$(pwd)" + SEPARATOR_GIVEN=0 + + while [ "$#" -gt 0 ]; do + case "$1" in + --) + SEPARATOR_GIVEN=1 + return + ;; + --data-root | --supercluster-root) + DATA_ROOT="$2" + ;; + --data-root=* | --supercluster-root=*) + DATA_ROOT="${1#*=}" + ;; + esac + + shift + done +} + +scan_args "$@" + +# Add a separator if the caller didn't supply one +if [ "$SEPARATOR_GIVEN" -eq 0 ]; then + set -- "$@" -- +fi + +exec sh "$SLP_EVAL" \ + --network-size-limit "$NETWORK_SIZE_LIMIT" \ + --pubnet-data "$DATA_ROOT/data/$PUBNET_DATA_FILE" \ + "$@" \ + --loadgen-keys "$DATA_ROOT/data/$LOADGEN_KEYS_FILE" \ + --measure-e2e-latency diff --git a/scripts/slp_eval.sh b/scripts/slp_eval.sh index 1043af4a..dc371358 100644 --- a/scripts/slp_eval.sh +++ b/scripts/slp_eval.sh @@ -3,9 +3,10 @@ # SLP mixed-load evaluation wrapper. # # This script runs the MinBlockTimeMixed mission against a stellar-core image -# using the 2025-06-24 pubnet topology data and the fixed benchmark parameters -# below. It is intended to answer: "does this image sustain the selected mixed -# classic/Soroban load at the normal 5s ledger close target?" +# using the 2025-06-24 pubnet topology data (by default; see --pubnet-data) and +# the fixed benchmark parameters below. It is intended to answer: "does this +# image sustain the selected mixed classic/Soroban load at the normal 5s ledger +# close target?" # # Benchmark setup: # - One mission run is started for each selected Soroban load flag: @@ -15,7 +16,8 @@ # match current network conditions. The flag value supplies only the Soroban # TPS for that run, so total TPS is CLASSIC_TX_RATE + selected Soroban TPS. # - The mission uses MinBlockTimeMixed's MIXED_PREGEN_* overlay-only loadgen -# mode, simulated pubnet network delay, with NETWORK_SIZE_LIMIT nodes. +# mode, simulated pubnet network delay, with a configurable number of nodes +# (--network-size-limit, default 277). # - The block-time search range is intentionally narrow: # [MIN_BLOCK_TIME_MS, MAX_BLOCK_TIME_MS] = [4900, 5100]. With the mission's # 100ms binary-search threshold, this effectively evaluates the 5s target @@ -23,6 +25,9 @@ # - simulate-apply-duration is derived from SIMULATE_APPLY_BUDGET_MS and the # total TPS so the synthetic apply sleep budget remains roughly constant as # the requested Soroban rate changes. +# - Arguments after a trailing "--" are appended verbatim to every mission +# command line, so one-off flags can be added without editing this script. +# scripts/measure_e2e.sh is a preset built on top of that mechanism. # # Result interpretation: # - A zero exit and a "Minimum sustainable block time: ..." log line means the @@ -41,6 +46,7 @@ STELLAR_CORE_IMAGE= SAC_TX_RATE= OZ_TX_RATE= SOROSWAP_TX_RATE= +PUBNET_DATA= IMAGE_REPOSITORY="746476062914.dkr.ecr.us-east-1.amazonaws.com/dev" @@ -64,11 +70,15 @@ NUM_PREGENERATED_TXS=1000000 GENESIS_TEST_ACCOUNT_COUNT=1000000 SIMULATE_APPLY_WEIGHT=100 SIMULATE_APPLY_BUDGET_MS=600 -NETWORK_SIZE_LIMIT=277 +DEFAULT_NETWORK_SIZE_LIMIT=277 +NETWORK_SIZE_LIMIT="$DEFAULT_NETWORK_SIZE_LIMIT" +DEFAULT_PUBNET_DATA_FILE="public-network-data-2025-06-24.json" usage() { cat </data/${DEFAULT_PUBNET_DATA_FILE}. --sac RATE Run SAC load with the given Soroban tx rate. Can be supplied with other load flags to run benchmarks sequentially. --oz RATE Run OZ load with the given Soroban tx rate. @@ -85,12 +99,13 @@ Options: --soroswap RATE Run Soroswap load with the given Soroban tx rate. Can be supplied with other load flags to run benchmarks sequentially. -h, --help Show this help. + -- MISSION_ARG... Everything after "--" is appended verbatim to each + mission command line. Use it for one-off flags this + wrapper does not expose. Benchmark constants: Classic TPS: ${CLASSIC_TX_RATE} Target close time: ${BLOCK_TIME_MS}ms, evaluated via [${MIN_BLOCK_TIME_MS}, ${MAX_BLOCK_TIME_MS}] - Network size limit: ${NETWORK_SIZE_LIMIT} - Data set: data/public-network-data-2025-06-24.json Results: PASS: command exits 0 and logs "Minimum sustainable block time: ...". @@ -112,7 +127,13 @@ is_nonnegative_integer() { esac } +# Parses the wrapper's own options and stops at a "--" separator. Sets +# PARSED_ARG_COUNT to the number of arguments consumed (including the +# separator) so the caller can shift them off and keep the mission arguments +# that followed in "$@". parse_args() { + total_arg_count="$#" + while [ "$#" -gt 0 ]; do case "$1" in --stellar-core-image | --image) @@ -180,6 +201,43 @@ parse_args() { SOROSWAP_TX_RATE="${1#*=}" shift ;; + --network-size-limit) + if [ "$#" -lt 2 ] || [ -z "$2" ]; then + printf '%s\n' "--network-size-limit requires a node count." >&2 + usage >&2 + exit 1 + fi + NETWORK_SIZE_LIMIT="$2" + shift 2 + ;; + --network-size-limit=*) + NETWORK_SIZE_LIMIT="${1#*=}" + shift + ;; + --pubnet-data) + if [ "$#" -lt 2 ] || [ -z "$2" ]; then + printf '%s\n' "--pubnet-data requires a path." >&2 + usage >&2 + exit 1 + fi + PUBNET_DATA="$2" + shift 2 + ;; + --pubnet-data=*) + PUBNET_DATA="${1#*=}" + # An empty value here would silently fall back to the + # default data set, so reject it like the two-token form. + if [ -z "$PUBNET_DATA" ]; then + printf '%s\n' "--pubnet-data requires a path." >&2 + usage >&2 + exit 1 + fi + shift + ;; + --) + shift + break + ;; -h | --help) usage exit 0 @@ -191,6 +249,8 @@ parse_args() { ;; esac done + + PARSED_ARG_COUNT=$((total_arg_count - $#)) } validate_tx_rate() { @@ -227,6 +287,13 @@ validate_args() { if [ -n "$SOROSWAP_TX_RATE" ]; then validate_tx_rate "--soroswap" "$SOROSWAP_TX_RATE" fi + + # Non-numeric values make "[" exit 2 rather than 1, so test for the + # accepted range and negate: anything unparseable is rejected too. + if ! [ "$NETWORK_SIZE_LIMIT" -ge 1 ] 2>/dev/null; then + printf '%s\n' "--network-size-limit must be a positive integer." >&2 + exit 1 + fi } calculate_simulate_apply_duration() { @@ -266,8 +333,9 @@ resolve_min_block_time_mixed_mode() { } run_min_block_time_mixed() { - mode_alias="${1:?usage: run_min_block_time_mixed MODE SOROBAN_TX_RATE}" - soroban_tx_rate="${2:?usage: run_min_block_time_mixed MODE SOROBAN_TX_RATE}" + mode_alias="${1:?usage: run_min_block_time_mixed MODE SOROBAN_TX_RATE [MISSION_ARG...]}" + soroban_tx_rate="${2:?usage: run_min_block_time_mixed MODE SOROBAN_TX_RATE [MISSION_ARG...]}" + shift 2 min_block_time_mixed_mode="$(resolve_min_block_time_mixed_mode "$mode_alias")" simulate_apply_duration="$(calculate_simulate_apply_duration "$CLASSIC_TX_RATE" "$soroban_tx_rate")" @@ -296,23 +364,28 @@ run_min_block_time_mixed() { --tier1-keys "$TIER1_KEYS" \ --network-size-limit "$NETWORK_SIZE_LIMIT" \ --require-node-labels=purpose:largetests \ - --tolerate-node-taints=largetests + --tolerate-node-taints=largetests \ + "$@" } parse_args "$@" +shift "$PARSED_ARG_COUNT" validate_args -PUBNET_DATA="$DATA_ROOT/data/public-network-data-2025-06-24.json" +if [ -z "$PUBNET_DATA" ]; then + PUBNET_DATA="$DATA_ROOT/data/$DEFAULT_PUBNET_DATA_FILE" +fi + TIER1_KEYS="$DATA_ROOT/data/tier1keys.json" if [ -n "$SAC_TX_RATE" ]; then - run_min_block_time_mixed sac "$SAC_TX_RATE" + run_min_block_time_mixed sac "$SAC_TX_RATE" "$@" fi if [ -n "$OZ_TX_RATE" ]; then - run_min_block_time_mixed oz "$OZ_TX_RATE" + run_min_block_time_mixed oz "$OZ_TX_RATE" "$@" fi if [ -n "$SOROSWAP_TX_RATE" ]; then - run_min_block_time_mixed soroswap "$SOROSWAP_TX_RATE" + run_min_block_time_mixed soroswap "$SOROSWAP_TX_RATE" "$@" fi diff --git a/src/App/Program.fs b/src/App/Program.fs index 0dd5125a..4ff9649f 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -73,8 +73,10 @@ type MissionOptions apiRateLimit: int, httpProxyReplicas: int, pubnetData: string option, + measureE2eLatency: bool, flatQuorum: bool option, tier1Keys: string option, + loadgenKeys: string option, maxConnections: int option, fullyConnectTier1: bool, byteCountValues: seq, @@ -318,12 +320,21 @@ type MissionOptions [] member self.PubnetData = pubnetData + [] + member self.MeasureE2eLatency = measureE2eLatency + [] member self.FlatQuorum = flatQuorum [] member self.Tier1Keys = tier1Keys + [] + member self.LoadgenKeys = loadgenKeys + [] @@ -743,6 +754,12 @@ let main argv = 0 | :? MissionOptions as mission -> + if mission.LoadgenKeys.IsSome && mission.PubnetData.IsNone then + failwith "Error: --loadgen-keys requires --pubnet-data to be set" + + if mission.MeasureE2eLatency && mission.LoadgenKeys.IsNone then + failwith "Error: --measure-e2e-latency requires --loadgen-keys" + let _ = logToConsoleAndFile (sprintf "%s/stellar-supercluster.log" mission.Destination) let ll = @@ -848,8 +865,10 @@ let main argv = apiRateLimit = mission.ApiRateLimit httpProxyReplicas = mission.HttpProxyReplicas pubnetData = mission.PubnetData + measureE2eLatency = mission.MeasureE2eLatency flatQuorum = mission.FlatQuorum tier1Keys = mission.Tier1Keys + loadgenKeys = mission.LoadgenKeys maxConnections = mission.MaxConnections fullyConnectTier1 = mission.FullyConnectTier1 byteCountDistribution = diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 7a3c4f73..f7e76624 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -71,8 +71,10 @@ let ctx : MissionContext = apiRateLimit = 10 httpProxyReplicas = 2 pubnetData = None + measureE2eLatency = false flatQuorum = None tier1Keys = None + loadgenKeys = None maxConnections = None fullyConnectTier1 = false peerReadingCapacity = None diff --git a/src/FSLibrary/MaxTPSTest.fs b/src/FSLibrary/MaxTPSTest.fs index b967a9ff..19bc5c3d 100644 --- a/src/FSLibrary/MaxTPSTest.fs +++ b/src/FSLibrary/MaxTPSTest.fs @@ -165,9 +165,15 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG List.find (fun (cs: CoreSet) -> cs.name.StringName = "stellar" || cs.name.StringName = "sdf") allNodes let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad) allNodes + + let loadGenNodes = + if List.isEmpty loadGenNodes then + // On smaller networks, run loadgen on all nodes to better balance the overhead of load generation + if List.length allNodes > smallNetworkSize then tier1 else allNodes + else + loadGenNodes - // On smaller networks, run loadgen on all nodes to better balance the overhead of load generation - let loadGenNodes = if List.length allNodes > smallNetworkSize then tier1 else allNodes let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes // Assign pre-generated transaction information to each load generator node. diff --git a/src/FSLibrary/MinBlockTimeTest.fs b/src/FSLibrary/MinBlockTimeTest.fs index 571cb427..0ec8f8e2 100644 --- a/src/FSLibrary/MinBlockTimeTest.fs +++ b/src/FSLibrary/MinBlockTimeTest.fs @@ -255,6 +255,32 @@ let private collectLedgerAgePercentiles |> Async.RunSynchronously |> Array.toList +let private logE2eLatencyMetrics (formation: StellarFormation) (coreSets: CoreSet list) : unit = + let e2eLatencyMetrics : (string * (Metrics.Metrics -> Metrics.GenericCounter option)) list = + [ "min", (fun m -> m.LoadgenTxLatencyRunMinMs) + "mean", (fun m -> m.LoadgenTxLatencyRunMeanMs) + "p50", (fun m -> m.LoadgenTxLatencyRunP50Ms) + "p75", (fun m -> m.LoadgenTxLatencyRunP75Ms) + "p99", (fun m -> m.LoadgenTxLatencyRunP99Ms) + "max", (fun m -> m.LoadgenTxLatencyRunMaxMs) ] + + for peer in formation.NetworkCfg.PeersInSets(List.toArray coreSets) do + let metrics = peer.GetMetrics() + + let summary = + e2eLatencyMetrics + |> List.map + (fun (name, get) -> + let value = + get metrics + |> Option.map (fun c -> string c.Count) + |> Option.defaultValue "none" + + sprintf "%s=%s" name value) + |> String.concat " " + + LogInfo "TX e2e latency: peer=%s loadgen-tx-latency-run-ms: %s" peer.ShortName.StringName summary + // Returns true iff every peer's ledger.age.closed-histogram satisfies: // P75 in [0.80*T, 1.20*T) // P99 <= 2*T @@ -337,8 +363,13 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: None } let tier1 = List.filter (fun (cs: CoreSet) -> cs.options.tier1 = Some true) allNodes + let loadGenNodes = List.filter (fun (cs: CoreSet) -> cs.options.generatesLoad) allNodes - let loadGenNodes = if List.length allNodes > smallNetworkSize then tier1 else allNodes + let loadGenNodes = + if List.isEmpty loadGenNodes then + if List.length allNodes > smallNetworkSize then tier1 else allNodes + else + loadGenNodes let isLoadGenNode cs = List.exists (fun (cs': CoreSet) -> cs' = cs) loadGenNodes @@ -496,6 +527,9 @@ let minBlockTimeTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: // long enough to skew the ledger age percentiles. let ledgerAgePercentiles = collectLedgerAgePercentiles formation allNodes + if context.measureE2eLatency then + logE2eLatencyMetrics formation activeLoadGenNodes + formation.CheckNoErrorsAndPairwiseConsistency() formation.EnsureAllNodesInSync allNodes checkLedgerAgeSLA ledgerAgePercentiles targetMs diff --git a/src/FSLibrary/StellarCoreCfg.fs b/src/FSLibrary/StellarCoreCfg.fs index 190fea2b..928c767c 100644 --- a/src/FSLibrary/StellarCoreCfg.fs +++ b/src/FSLibrary/StellarCoreCfg.fs @@ -166,6 +166,7 @@ type StellarCoreCfg = automaticMaintenanceCount: int accelerateTime: bool generateLoad: bool + measureE2eLatency: bool updateSorobanCosts: bool option manualClose: bool invariantChecks: InvariantChecksSpec @@ -317,6 +318,9 @@ type StellarCoreCfg = t.Add("ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING", self.accelerateTime) |> ignore t.Add("ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING", self.generateLoad) |> ignore + if self.measureE2eLatency && self.network.missionContext.measureE2eLatency then + t.Add("LOADGEN_MEASURE_TX_E2E_LATENCY_FOR_TESTING", true) |> ignore + if self.updateSorobanCosts.IsSome then t.Add("UPDATE_SOROBAN_COSTS_DURING_PROTOCOL_UPGRADE_FOR_TESTING", self.updateSorobanCosts.Value) |> ignore @@ -653,6 +657,7 @@ type NetworkCfg with automaticMaintenanceCount = if opts.performMaintenance then 50000 else 0 accelerateTime = opts.accelerateTime generateLoad = true + measureE2eLatency = opts.generatesLoad updateSorobanCosts = opts.updateSorobanCosts manualClose = false invariantChecks = opts.invariantChecks @@ -698,6 +703,7 @@ type NetworkCfg with automaticMaintenanceCount = if c.options.performMaintenance then 50000 else 0 accelerateTime = c.options.accelerateTime generateLoad = true + measureE2eLatency = c.options.generatesLoad updateSorobanCosts = c.options.updateSorobanCosts manualClose = false invariantChecks = c.options.invariantChecks diff --git a/src/FSLibrary/StellarCoreSet.fs b/src/FSLibrary/StellarCoreSet.fs index e1c1c3ca..da28bda5 100644 --- a/src/FSLibrary/StellarCoreSet.fs +++ b/src/FSLibrary/StellarCoreSet.fs @@ -211,6 +211,7 @@ type CoreSetOptions = validate: bool homeDomain: string option tier1: bool option + generatesLoad: bool catchupMode: CatchupMode image: string initialization: CoreSetInitialization @@ -254,6 +255,7 @@ type CoreSetOptions = validate = true homeDomain = Some "stellar.org" tier1 = None + generatesLoad = false catchupMode = CatchupComplete image = image initialization = CoreSetInitialization.Default diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 828048ff..f94b972a 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -81,8 +81,10 @@ type MissionContext = apiRateLimit: int httpProxyReplicas: int pubnetData: string option + measureE2eLatency: bool flatQuorum: bool option tier1Keys: string option + loadgenKeys: string option maxConnections: int option fullyConnectTier1: bool byteCountDistribution: ((int * int) list) diff --git a/src/FSLibrary/StellarNetworkData.fs b/src/FSLibrary/StellarNetworkData.fs index 61f3d6b1..86a94c05 100644 --- a/src/FSLibrary/StellarNetworkData.fs +++ b/src/FSLibrary/StellarNetworkData.fs @@ -20,9 +20,33 @@ let PubnetLatestHistoryArchiveState = let TestnetLatestHistoryArchiveState = "http://history.stellar.org/prd/core-testnet/core_testnet_001/.well-known/stellar-history.json" -type PubnetNode = JsonProvider<"json-type-samples/sample-network-data.json", SampleIsList=false, ResolutionFolder=cwd> +type PubnetNodeJSON = + JsonProvider<"json-type-samples/sample-network-data.json", SampleIsList=false, ResolutionFolder=cwd> + type Tier1PublicKey = JsonProvider<"json-type-samples/sample-keys.json", SampleIsList=false, ResolutionFolder=cwd> +// Using an actual record instead of the JSON type allows us to use record +// update syntax. The static member also lets us verify earlier that the JSON is +// of the correct shape. +type PubnetNode = + { PublicKey: string + Peers: string array + RadarHomeDomain: string option + RadarIsValidating: bool option + RadarGeoData: {| Latitude: decimal; Longitude: decimal |} option + RadarName: string option } + + static member ofJSON(node: PubnetNodeJSON.Root) : PubnetNode = + { PublicKey = node.PublicKey + Peers = node.Peers + RadarHomeDomain = node.RadarHomeDomain + RadarIsValidating = node.RadarIsValidating + RadarGeoData = + match node.RadarGeoData with + | Some geoData -> Some {| Latitude = geoData.Latitude; Longitude = geoData.Longitude |} + | None -> None + RadarName = node.RadarName } + // Adjacency map for peers type PeerMap = Map> @@ -147,8 +171,8 @@ let locations = // Each edge connecting a and b is represented as (a, b) if a < b, and (b, a) otherwise. // This makes sense as the graph is undirected, and it also makes it easier to handle a set of edges. -let extractEdges (graph: PubnetNode.Root array) : (string * string) array = - let getEdgesFromNode (node: PubnetNode.Root) : (string * string) array = +let extractEdges (graph: PubnetNode array) : (string * string) array = + let getEdgesFromNode (node: PubnetNode) : (string * string) array = node.Peers |> Array.filter (fun peer -> peer < node.PublicKey) // This filter ensures that we add each edge exactly once. |> Array.map (fun peer -> (peer, node.PublicKey)) @@ -260,7 +284,7 @@ let private pruneAdjacencyMap (maxConnections: int) (noPrune: Set) (m: P // then we pick a random edge (a, b), remove (a, b) and add (a, u) and (u, b). // We continue this process until u has a desired degree. let addEdges - (graph: PubnetNode.Root array) + (graph: PubnetNode array) (newNodes: string array) (tier1KeySet: Set) (random: System.Random) @@ -334,41 +358,56 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin if context.pubnetData.IsNone then failwith "pubnet simulation requires --pubnet-data=" - let allPubnetNodes : PubnetNode.Root array = PubnetNode.Load(context.pubnetData.Value) + let allPubnetNodes : PubnetNode array = PubnetNodeJSON.Load(context.pubnetData.Value) |> Array.map PubnetNode.ofJSON // A Random object with a fixed seed. let random = System.Random context.randomSeed let newTier1Nodes = [ for i in 1 .. context.tier1OrgsToAdd * tier1OrgSize -> - PubnetNode.Parse( + PubnetNodeJSON.Parse( sprintf """ [{ "publicKey": "%s", "radar_homeDomain": "home.domain.%d" }] """ (KeyPair.Random().Address) ((i - 1) / tier1OrgSize) - ).[0] ] + ).[0] + |> PubnetNode.ofJSON ] |> Array.ofList let newNonTier1Nodes = [ for i in 1 .. context.nonTier1NodesToAdd -> - PubnetNode.Parse(sprintf """ [{ "publicKey": "%s" }] """ (KeyPair.Random().Address)).[0] ] + PubnetNodeJSON.Parse(sprintf """ [{ "publicKey": "%s" }] """ (KeyPair.Random().Address)).[0] + |> PubnetNode.ofJSON ] |> Array.ofList let tier1KeySet : Set = if context.tier1Keys.IsSome then - let newTier1Keys = Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) newTier1Nodes in + let newTier1Keys = Array.map (fun (n: PubnetNode) -> n.PublicKey) newTier1Nodes in Tier1PublicKey.Load(context.tier1Keys.Value) |> Array.map (fun n -> n.PublicKey) |> Array.append newTier1Keys |> Set.ofArray else - PubnetNode.Load(context.pubnetData.Value) + PubnetNodeJSON.Load(context.pubnetData.Value) // Any node with a home domain is considered tier1. |> Array.filter (fun n -> n.RadarHomeDomain.IsSome) |> Array.map (fun n -> n.PublicKey) |> Set.ofArray + let loadgenKeySet : Set = + if context.loadgenKeys.IsSome then + Tier1PublicKey.Load(context.loadgenKeys.Value) + |> Array.map (fun n -> n.PublicKey) + |> Set.ofArray + else + Set.empty + + // Check that there is no overlap between tier1 and loadgen keys + let overlap = Set.intersect tier1KeySet loadgenKeySet + + if not (Set.isEmpty overlap) then + failwithf "Overlap between tier1 and loadgen keys: %A" overlap // Shuffle the nodes to ensure that the order will // not affect the outcome of the scaling algorithm. @@ -376,7 +415,16 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin Array.append newTier1Nodes newNonTier1Nodes |> Array.sortBy (fun _ -> random.Next()) - let allPubnetNodes = allPubnetNodes |> Array.append newNodes + let allPubnetNodes = + allPubnetNodes + |> Array.append newNodes + // Ensure that each load-generator has a home domain + |> Array.map + (fun n -> + if Set.contains n.PublicKey loadgenKeySet && n.RadarHomeDomain.IsNone then + { n with RadarHomeDomain = Some "loadgennode" } + else + n) // For each pubkey in the pubnet, we map it to an actual KeyPair (with a private // key) to use in the simulation. It's important to keep these straight! The keys @@ -384,7 +432,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // throughout the rest of this function, as strings called "pubkey", but should not // appear in the final CoreSets we're building. let mutable pubnetKeyToSimKey : Map = - Array.map (fun (n: PubnetNode.Root) -> (n.PublicKey, KeyPair.Random())) allPubnetNodes + Array.map (fun (n: PubnetNode) -> (n.PublicKey, KeyPair.Random())) allPubnetNodes |> Map.ofArray // Not every pubkey used in the qsets is represented in the base set of pubnet nodes, @@ -401,7 +449,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // This is because `networkSizeLimit` may be smaller than the degree of some new node // and cause an issue to the scaling algorithm. let adjacencyMap = - addEdges allPubnetNodes (Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) newNodes) tier1KeySet random + addEdges allPubnetNodes (Array.map (fun (n: PubnetNode) -> n.PublicKey) newNodes) tier1KeySet random |> if context.fullyConnectTier1 then fullyConnectTier1 tier1KeySet else id |> match context.maxConnections with | Some maxConnections -> @@ -420,8 +468,8 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let orgNodes, miscNodes = allPubnetNodes - |> Array.filter (fun (n: PubnetNode.Root) -> minAllowedConnectionCount <= numPeers adjacencyMap n.PublicKey) - |> Array.partition (fun (n: PubnetNode.Root) -> n.RadarHomeDomain.IsSome) + |> Array.filter (fun (n: PubnetNode) -> minAllowedConnectionCount <= numPeers adjacencyMap n.PublicKey) + |> Array.partition (fun (n: PubnetNode) -> n.RadarHomeDomain.IsSome) // We then trim down the set of misc nodes so that they fit within simulation // size limit passed. If we can't even fit the org nodes, we fail here. @@ -439,18 +487,33 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let allPubnetNodes = Array.append orgNodes miscNodes let _ = assert ((Array.length allPubnetNodes) <= context.networkSizeLimit) - let allPubnetNodeKeys = - Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) allPubnetNodes - |> Set.ofArray + let allPubnetNodeKeys = Array.map (fun (n: PubnetNode) -> n.PublicKey) allPubnetNodes |> Set.ofArray + + // Check if we removed any tier1 or loadgen nodes. + let keptNodes : Set = + allPubnetNodeKeys + |> Seq.filter (fun (n: string) -> Set.contains n tier1KeySet || Set.contains n loadgenKeySet) + |> Set.ofSeq + + for removedNode in Set.difference tier1KeySet keptNodes do + LogWarn "Removed tier1 node %s from simulation" removedNode + + for removedNode in Set.difference loadgenKeySet keptNodes do + LogWarn "Removed loadgen node %s from simulation" removedNode LogInfo "SimulatePubnet will run with %d nodes" (Array.length allPubnetNodes) // We then group the org nodes by their home domains. The domain names are drawn // from the HomeDomains of the public network but with periods replaced with dashes, // and lowercased, so for example keybase.io turns into keybase-io. - let groupedOrgNodes : (HomeDomainName * PubnetNode.Root array) array = + // Additionally, each load generator needs to exist in a size-1 org (because + // the downstream only generates load on one node per CoreSet). So, we add a + // loadgen-%d suffix to each load generator. + let loadgenNumber : Map = loadgenKeySet |> Seq.mapi (fun i key -> key, i) |> Map.ofSeq + + let groupedOrgNodes : (HomeDomainName * PubnetNode array) array = Array.groupBy - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> let domain = n.RadarHomeDomain.Value // We turn 'www.stellar.org' into 'stellar' // and 'stellar.blockdaemon.com' into 'blockdaemon' @@ -467,23 +530,35 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // or not tier 1, we split them into separate orgs by appending // "-non-tier1" to the home domain of any non-tier1 node in an // org. - let withTierInfo = - if not (Set.contains n.PublicKey tier1KeySet) then + let withNodeInfo = + if Set.contains n.PublicKey loadgenKeySet then + cleanOrgName + "-loadgen-" + loadgenNumber.[n.PublicKey].ToString() + elif not (Set.contains n.PublicKey tier1KeySet) then cleanOrgName + "-non-tier1" else cleanOrgName - let lowercase = withTierInfo.ToLower() + let lowercase = withNodeInfo.ToLower() HomeDomainName lowercase) orgNodes + // Check that load generators each exist in size-1 orgs + for hdn, nodes in groupedOrgNodes do + let loadGenerators = nodes |> Array.filter (fun n -> Set.contains n.PublicKey loadgenKeySet) + + if loadGenerators.Length > 0 && nodes.Length > 1 then + failwithf + "Load generator node(s) found in org %s with %d nodes. Each node that generates load should have its own home domain" + hdn.StringName + nodes.Length + // Then build a map from accountID to HomeDomainName and index-within-domain // for each org node. let orgNodeHomeDomains : Map = Array.collect - (fun (hdn: HomeDomainName, nodes: PubnetNode.Root array) -> - Array.mapi (fun (i: int) (n: PubnetNode.Root) -> (n.PublicKey, (hdn, i))) nodes) + (fun (hdn: HomeDomainName, nodes: PubnetNode array) -> + Array.mapi (fun (i: int) (n: PubnetNode) -> (n.PublicKey, (hdn, i))) nodes) groupedOrgNodes |> Map.ofArray @@ -538,14 +613,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let defaultQuorum : QuorumSetSpec = let tier1Nodes = allPubnetNodes - |> Array.filter - (fun (n: PubnetNode.Root) -> (Set.contains n.PublicKey tier1KeySet) && n.RadarHomeDomain.IsSome) + |> Array.filter (fun (n: PubnetNode) -> (Set.contains n.PublicKey tier1KeySet) && n.RadarHomeDomain.IsSome) let tier1NodesGroupedByHomeDomain : (string array) array = tier1Nodes - |> Array.groupBy (fun (n: PubnetNode.Root) -> n.RadarHomeDomain.Value) - |> Array.map - (fun (_, nodes: PubnetNode.Root []) -> Array.map (fun (n: PubnetNode.Root) -> n.PublicKey) nodes) + |> Array.groupBy (fun (n: PubnetNode) -> n.RadarHomeDomain.Value) + |> Array.map (fun (_, nodes: PubnetNode []) -> Array.map (fun (n: PubnetNode) -> n.PublicKey) nodes) let orgToExplicitQSet (org: string array) : ExplicitQuorumSet = { thresholdPercent = Some(51) // Simple majority @@ -555,7 +628,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let tier1Orgs = tier1Nodes |> Array.map - (fun (n: PubnetNode.Root) -> { name = (homeDomainNameForKey n.PublicKey).StringName; quality = High }) + (fun (n: PubnetNode) -> { name = (homeDomainNameForKey n.PublicKey).StringName; quality = High }) |> Set.ofArray let flatQset = @@ -596,9 +669,9 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // as long as the random function is persistent. let geoLocations : GeoLoc array = allPubnetNodes - |> Array.filter (fun (n: PubnetNode.Root) -> n.RadarGeoData.IsSome) + |> Array.filter (fun (n: PubnetNode) -> n.RadarGeoData.IsSome) |> Array.map - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> { lat = float n.RadarGeoData.Value.Latitude lon = float n.RadarGeoData.Value.Longitude }) |> Seq.ofArray @@ -611,7 +684,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // The assignment is deterministic as it depends on the public key of the // node. This ensures that geolocations persist across runs, even if the // total number of nodes changes via the *-orgs-to-add flags. - let getGeoLocOrDefault (n: PubnetNode.Root) : GeoLoc = + let getGeoLocOrDefault (n: PubnetNode) : GeoLoc = match n.RadarGeoData with | Some geoData -> { lat = float geoData.Latitude; lon = float geoData.Longitude } | None -> @@ -636,7 +709,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin allPubnetNodes |> Array.map - (fun (n: PubnetNode.Root) -> + (fun (n: PubnetNode) -> let key = getSimPubKey n.PublicKey let peers = @@ -659,7 +732,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin // Given a node, returns a tuple where the first element is a boolean // indicating whether the node is a validator, and the second element is an // appropriate quorum set configuration for that node. - let computeQset (n: PubnetNode.Root) = + let computeQset (n: PubnetNode) = let tier1 = Set.contains n.PublicKey tier1KeySet let hdn = homeDomainNameForKey n.PublicKey @@ -746,7 +819,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let miscCoreSets : CoreSet array = Array.mapi - (fun (_: int) (n: PubnetNode.Root) -> + (fun (_: int) (n: PubnetNode) -> let hdn = homeDomainNameForKey n.PublicKey let keys = [| getSimKey n.PublicKey |] let tier1 = Set.contains n.PublicKey tier1KeySet @@ -760,6 +833,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None + generatesLoad = false nodeLocs = Some [ getGeoLocOrDefault n ] preferredPeersMap = Some(keysToPreferredPeersMap keys) } @@ -770,11 +844,12 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin let orgCoreSets : CoreSet array = Array.map - (fun (hdn: HomeDomainName, nodes: PubnetNode.Root array) -> + (fun (hdn: HomeDomainName, nodes: PubnetNode array) -> assert (nodes.Length <> 0) let nodeList = List.ofArray nodes - let keys = Array.map (fun (n: PubnetNode.Root) -> getSimKey n.PublicKey) nodes + let keys = Array.map (fun (n: PubnetNode) -> getSimKey n.PublicKey) nodes let tier1 = Set.contains nodes.[0].PublicKey tier1KeySet + let generatesLoad = Set.contains nodes.[0].PublicKey loadgenKeySet let validate, qset = mergeQSets (List.map computeQset nodeList) @@ -785,6 +860,7 @@ let FullPubnetCoreSets (context: MissionContext) (manualclose: bool) (enforceMin tier1 = Some tier1 validate = validate homeDomain = if validate then Some hdn.StringName else None + generatesLoad = generatesLoad nodeLocs = Some(List.map getGeoLocOrDefault nodeList) preferredPeersMap = Some(keysToPreferredPeersMap keys) } diff --git a/src/FSLibrary/json-type-samples/sample-metrics.json b/src/FSLibrary/json-type-samples/sample-metrics.json index cc01a8e8..bdc4c681 100644 --- a/src/FSLibrary/json-type-samples/sample-metrics.json +++ b/src/FSLibrary/json-type-samples/sample-metrics.json @@ -3533,6 +3533,30 @@ "5_min_rate": 6.91049e-08, "15_min_rate": 6.91049e-08 }, + "loadgen.tx-latency-run.max-ms": { + "type": "counter", + "count": 14797 + }, + "loadgen.tx-latency-run.mean-ms": { + "type": "counter", + "count": 7573 + }, + "loadgen.tx-latency-run.min-ms": { + "type": "counter", + "count": 1947 + }, + "loadgen.tx-latency-run.p50-ms": { + "type": "counter", + "count": 7482 + }, + "loadgen.tx-latency-run.p75-ms": { + "type": "counter", + "count": 9334 + }, + "loadgen.tx-latency-run.p99-ms": { + "type": "counter", + "count": 12706 + }, "overlay.byte.read": { "type": "meter", "count": 5,