From 627871db18b91acb61066645a55881fb370324ef Mon Sep 17 00:00:00 2001 From: ecency Date: Fri, 17 Jul 2026 06:03:29 +0000 Subject: [PATCH 1/4] fix: per-fetch caps so slow upstreams degrade one wallet, not a whole layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit portfolioV2 runs its chain and engine layers under a single leg timeout. Chain balance providers allow 10s per attempt and the rewards upstream 30s, so one cold or slow provider blew the leg budget and the entire layer silently vanished from the response — observed intermittently in production while every individual balance endpoint answered fast. - Chain: each wallet's balance fetch is capped at 2s; a capped wallet becomes an error item instead of sinking its siblings. Two concurrency batches still fit the leg budget. - Engine: the unclaimed-rewards fetch is capped at 2s and degrades to no pending-reward badges instead of blanking the token list. --- dotnet/EcencyApi/Handlers/WalletApi.Engine.cs | 7 ++++++- dotnet/EcencyApi/Handlers/WalletApi.Layers.cs | 14 +++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs b/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs index 01f8a26d..595b0860 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Engine.cs @@ -165,7 +165,11 @@ private static async Task FetchEngineTokensWithBalance(string usernam var tokensTask = FetchEngineTokens(symbols); var metricsTask = FetchEngineMetrics(symbols); - var unclaimedTask = FetchEngineRewards(username); + // The rewards upstream allows 30s, but this whole fetch must fit + // the portfolioV2 engine leg budget (4.5s) — a slow rewards call + // would otherwise blank the entire engine layer. Rewards are + // decoration (pendingToken badges); degrade to none instead. + var unclaimedTask = WithTimeout(FetchEngineRewards(username), EngineRewardsTimeoutMs, new JsonArray()); await Task.WhenAll(tokensTask, metricsTask, unclaimedTask); var tokens = tokensTask.Result; var metrics = metricsTask.Result; @@ -295,6 +299,7 @@ await Task.WhenAll(globalPropsTask, accountTask, marketTask, pointsTask, engineT private const int FastLegTimeout = 3000; private const int SlowLegTimeout = 4500; + private const int EngineRewardsTimeoutMs = 2000; private static async Task WithTimeout(Task task, int ms, T fallback) { diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs index 28ecbf12..77323190 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs @@ -659,6 +659,15 @@ internal static async Task> ProcessWithConcurrencyLimit(IReadOnlyL return results.ToList(); } + // Per-wallet cap on the balance fetch. The chain providers allow 10s per + // attempt, but the portfolioV2 chain leg has a 4.5s budget for the WHOLE + // layer — without a per-wallet cap, one cold/slow provider blows the leg + // timeout and the entire chain layer silently disappears from the response + // (observed intermittently in production). 2s keeps two concurrency + // batches inside the leg budget; the capped wallet degrades to an error + // item instead of sinking its siblings. + private const int ChainBalanceTimeoutMs = 2000; + internal static async Task> BuildChainLayer(JsonNode? accountData, JsonNode? marketData, string currency, bool? onlyEnabled) { var wallets = ExtractExternalWallets(accountData, onlyEnabled == true); @@ -672,7 +681,10 @@ internal static async Task> BuildChainLayer(JsonNode? accountDa try { - var data = await PrivateApi.FetchChainBalance(chain, wallet.Address); + var fetchTask = PrivateApi.FetchChainBalance(chain, wallet.Address); + var completed = await Task.WhenAny(fetchTask, Task.Delay(ChainBalanceTimeoutMs)); + if (completed != fetchTask) throw new Exception("Chain balance request timed out"); + var data = await fetchTask; var balance = ConvertChainBalanceToAmount(data, decimals); var price = GetTokenPrice(marketData, wallet.Symbol, currency); var iconUrl = config.IconUrl ?? Constants.AssetIconUrls["CHAIN_PLACEHOLDER"]; From e6f64d0b059d320452112c67fe5fd90795fa505f Mon Sep 17 00:00:00 2001 From: ecency Date: Fri, 17 Jul 2026 06:14:42 +0000 Subject: [PATCH 2/4] review: widen chain fetch concurrency so large pools fit the leg budget At width 3 a 7+ wallet account needed a third 2s batch and still blew the 4.5s leg. Width 8 covers 16 wallets in two worst-case batches (4s). --- dotnet/EcencyApi/Handlers/WalletApi.Layers.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs index 77323190..40d61644 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs @@ -663,10 +663,12 @@ internal static async Task> ProcessWithConcurrencyLimit(IReadOnlyL // attempt, but the portfolioV2 chain leg has a 4.5s budget for the WHOLE // layer — without a per-wallet cap, one cold/slow provider blows the leg // timeout and the entire chain layer silently disappears from the response - // (observed intermittently in production). 2s keeps two concurrency - // batches inside the leg budget; the capped wallet degrades to an error - // item instead of sinking its siblings. + // (observed intermittently in production). The capped wallet degrades to + // an error item instead of sinking its siblings. The concurrency width + // must keep ceil(n / width) * cap under the leg budget: at width 8, up to + // 16 wallets (2 batches) finish worst-case in 4s. private const int ChainBalanceTimeoutMs = 2000; + private const int ChainFetchConcurrency = 8; internal static async Task> BuildChainLayer(JsonNode? accountData, JsonNode? marketData, string currency, bool? onlyEnabled) { @@ -707,7 +709,7 @@ internal static async Task> BuildChainLayer(JsonNode? accountDa new ItemOptions { Address = wallet.Address, Error = errorMessage }, config.IconUrl ?? Constants.AssetIconUrls["CHAIN_PLACEHOLDER"], ChainActions()); } - }, 3); + }, ChainFetchConcurrency); return items.Where(x => x != null).ToList(); } From 7593fc6a91197ba8a07f81f17cab19c8227b662c Mon Sep 17 00:00:00 2001 From: ecency Date: Fri, 17 Jul 2026 06:20:05 +0000 Subject: [PATCH 3/4] review: hard-cap processed chain wallets at 16 Guarantees at most two fetch waves inside the leg budget regardless of how many addresses a profile lists; oversized profiles keep their first 16 wallets instead of risking the whole layer. --- dotnet/EcencyApi/Handlers/WalletApi.Layers.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs index 40d61644..9321cabc 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs @@ -669,11 +669,16 @@ internal static async Task> ProcessWithConcurrencyLimit(IReadOnlyL // 16 wallets (2 batches) finish worst-case in 4s. private const int ChainBalanceTimeoutMs = 2000; private const int ChainFetchConcurrency = 8; + // Hard bound so the worst case is exactly two waves (4s < 4.5s leg) no + // matter how many addresses a profile lists. A profile with more entries + // keeps its first 16 wallets instead of risking the whole layer. + private const int MaxChainWallets = 16; internal static async Task> BuildChainLayer(JsonNode? accountData, JsonNode? marketData, string currency, bool? onlyEnabled) { var wallets = ExtractExternalWallets(accountData, onlyEnabled == true); if (wallets.Count == 0) return new List(); + if (wallets.Count > MaxChainWallets) wallets = wallets.Take(MaxChainWallets).ToList(); var items = await ProcessWithConcurrencyLimit(wallets, async wallet => { From d4b0deddcd799de0e02c2b9f796055dee6c74835 Mon Sep 17 00:00:00 2001 From: ecency Date: Fri, 17 Jul 2026 06:23:57 +0000 Subject: [PATCH 4/4] review: skipped over-cap wallets surface as error items The response now acknowledges every configured wallet: entries past the 16-wallet fetch cap appear with a 'Wallet limit reached' error instead of being silently omitted. --- dotnet/EcencyApi/Handlers/WalletApi.Layers.cs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs index 9321cabc..3c65d6b6 100644 --- a/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs +++ b/dotnet/EcencyApi/Handlers/WalletApi.Layers.cs @@ -670,15 +670,17 @@ internal static async Task> ProcessWithConcurrencyLimit(IReadOnlyL private const int ChainBalanceTimeoutMs = 2000; private const int ChainFetchConcurrency = 8; // Hard bound so the worst case is exactly two waves (4s < 4.5s leg) no - // matter how many addresses a profile lists. A profile with more entries - // keeps its first 16 wallets instead of risking the whole layer. + // matter how many addresses a profile lists. Wallets past the cap are not + // fetched but still appear as error items, so the response acknowledges + // every configured wallet instead of silently omitting the tail. private const int MaxChainWallets = 16; internal static async Task> BuildChainLayer(JsonNode? accountData, JsonNode? marketData, string currency, bool? onlyEnabled) { var wallets = ExtractExternalWallets(accountData, onlyEnabled == true); if (wallets.Count == 0) return new List(); - if (wallets.Count > MaxChainWallets) wallets = wallets.Take(MaxChainWallets).ToList(); + var skipped = wallets.Skip(MaxChainWallets).ToList(); + if (skipped.Count > 0) wallets = wallets.Take(MaxChainWallets).ToList(); var items = await ProcessWithConcurrencyLimit(wallets, async wallet => { @@ -716,7 +718,21 @@ internal static async Task> BuildChainLayer(JsonNode? accountDa } }, ChainFetchConcurrency); - return items.Where(x => x != null).ToList(); + var result = items.Where(x => x != null).ToList(); + foreach (var wallet in skipped) + { + var chain = wallet.Chain.ToLowerInvariant(); + var config = ChainConfigs[chain]; + var decimals = (int)(wallet.Decimals ?? config.Decimals); + var price = GetTokenPrice(marketData, wallet.Symbol, currency); + result.Add(MakePortfolioItem( + !string.IsNullOrEmpty(wallet.Name) ? wallet.Name : config.Name, + !string.IsNullOrEmpty(wallet.Symbol) ? wallet.Symbol : config.Symbol, + "chain", 0, price, currency, decimals, + new ItemOptions { Address = wallet.Address, Error = "Wallet limit reached" }, + config.IconUrl ?? Constants.AssetIconUrls["CHAIN_PLACEHOLDER"], ChainActions())); + } + return result; } // ---- JS numeric string formatting ------------------------------------