diff --git a/src/@types/index.ts b/src/@types/index.ts index c4187f8a..08306dfd 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -986,6 +986,7 @@ export interface NetworkParameters { minTollUsdStr: string defaultTollUsdStr: string goldenTicketServerUrl: string + goldenTicketRetryInterval: number dao: { proposalFeeUsdStr: string voteThresholdUsdStr: string diff --git a/src/config/index.ts b/src/config/index.ts index 3d880e8f..f1bdc983 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -104,6 +104,7 @@ export const INITIAL_PARAMETERS: NetworkParameters = { minTollUsdStr: '0.2', defaultTollUsdStr: '0.2', goldenTicketServerUrl: 'http://localhost:3456/golden/ticket', + goldenTicketRetryInterval: 10 * ONE_MINUTE, messageRetentionDays: 7, messageMaxLength: 500, dao: { diff --git a/src/index.ts b/src/index.ts index 98a01ee4..7e57e3ab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,6 +79,9 @@ const daoPreCrackTxTypes = new Set([ let isReadyToJoinLatestValue = false let mustUseAdminCert = false +let nextGoldenTicketRetryAt = 0 +let isGoldenTicketRetry = false +let isGoldenTicketFetchInProgress = false const shardusSetup = (): void => { // SDK SETUP FUNCTIONS @@ -1575,27 +1578,52 @@ const shardusSetup = (): void => { isReadyToJoinLatestValue = false mustUseAdminCert = false + const hasExpiredGoldenTicket = adminCert && adminCert.certExp <= dapp.shardusGetTime() && adminCert?.goldenTicket === true // query admin cert from the golden ticket server - if (!isRequestedAdminCert && networkAccount && utils.isEqualOrNewerVersion('2.4.3', networkAccount.current.activeVersion)) { + if ( + (!isRequestedAdminCert || hasExpiredGoldenTicket) && + networkAccount && + utils.isEqualOrNewerVersion('2.4.3', networkAccount.current.activeVersion) && + dapp.shardusGetTime() >= nextGoldenTicketRetryAt && + !isGoldenTicketFetchInProgress + ) { + isGoldenTicketFetchInProgress = true try { - markRequestedAdminCert() // one-time request only - - const goldenTicket = await tryAndFetchGoldenTicket(publicKey, networkAccount, dapp) - if (goldenTicket) { - setAdminCertificate(goldenTicket) + const goldenTicketResult = await tryAndFetchGoldenTicket(publicKey, networkAccount, dapp, isGoldenTicketRetry) + if (goldenTicketResult.ticket) { + setAdminCertificate(goldenTicketResult.ticket) + markRequestedAdminCert() + isGoldenTicketRetry = false /* prettier-ignore */ - if (LiberdusFlags.VerboseLogs) console.log(`fetched golden ticket: ${Utils.safeStringify(goldenTicket)}`) + if (LiberdusFlags.VerboseLogs) console.log(`fetched golden ticket: ${Utils.safeStringify(goldenTicketResult.ticket)}`) nestedCountersInstance.countEvent('liberdus-staking', 'fetched golden ticket from server') console.log(`Admin certificate is set to `, adminCert) + } else if (goldenTicketResult.terminal) { + // An expired ticket otherwise keeps hasExpiredGoldenTicket true and + // bypasses isRequestedAdminCert on every isReadyToJoin invocation. + if (hasExpiredGoldenTicket) setAdminCertificate(null) + markRequestedAdminCert() + isGoldenTicketRetry = false + /* prettier-ignore */ + if (LiberdusFlags.VerboseLogs) console.log(`terminal golden ticket fetch error: ${goldenTicketResult.error}`) + nestedCountersInstance.countEvent('liberdus-staking', `terminal golden ticket fetch error: ${goldenTicketResult.error}`) } else { + const goldenTicketRetryInterval = networkAccount.current.goldenTicketRetryInterval || 10 * configs.ONE_MINUTE + nextGoldenTicketRetryAt = dapp.shardusGetTime() + goldenTicketRetryInterval + isGoldenTicketRetry = true /* prettier-ignore */ - if (LiberdusFlags.VerboseLogs) console.log(`no golden ticket available from server`) - nestedCountersInstance.countEvent('liberdus-staking', 'no golden ticket available from server') + if (LiberdusFlags.VerboseLogs) console.log(`no golden ticket available from server, retrying in ${goldenTicketRetryInterval}ms`) + nestedCountersInstance.countEvent('liberdus-staking', `no golden ticket available from server, retrying in ${goldenTicketRetryInterval}ms`) } } catch (e) { + const goldenTicketRetryInterval = networkAccount.current.goldenTicketRetryInterval || 10 * configs.ONE_MINUTE + nextGoldenTicketRetryAt = dapp.shardusGetTime() + goldenTicketRetryInterval + isGoldenTicketRetry = true /* prettier-ignore */ - if (logFlags.error) console.log(`Error fetching golden ticket: ${e.message}`) // non fatal + if (logFlags.error) console.log(`Error fetching golden ticket: ${e.message}; retrying in ${goldenTicketRetryInterval}ms`) // non fatal + } finally { + isGoldenTicketFetchInProgress = false } } console.log('is AdminCert set to ', adminCert) diff --git a/src/transactions/admin_certificate.ts b/src/transactions/admin_certificate.ts index 1885fb00..f1e005a4 100644 --- a/src/transactions/admin_certificate.ts +++ b/src/transactions/admin_certificate.ts @@ -22,6 +22,13 @@ export interface AdminCertResponse { cached?: boolean } +export interface GoldenTicketFetchResult { + ticket?: AdminCert + error?: string + retryable: boolean + terminal: boolean +} + export type PutAdminCertRequest = AdminCert export interface PutAdminCertResult { @@ -32,6 +39,33 @@ export interface PutAdminCertResult { export let adminCert: AdminCert = null export let isRequestedAdminCert: boolean = false +export function isTerminalGoldenTicketError(error?: string): boolean { + if (!error) return false + const normalizedError = error.toLowerCase() + return ( + normalizedError.includes('schema validation failed') || + normalizedError.includes('invalid request format') || + normalizedError.includes('public key not registered') || + normalizedError.includes('inactive') || + normalizedError.includes('validator not found') || + normalizedError.includes('nonce cannot be empty') || + normalizedError.includes('nonce must be') || + normalizedError.includes('port must be between') || + normalizedError.includes('signature owner does not match registered public key') || + normalizedError.includes('invalid signature') || + normalizedError.includes('signature validation failed') + ) +} + +function createGoldenTicketFetchResult(error?: string): GoldenTicketFetchResult { + const terminal = isTerminalGoldenTicketError(error) + return { + error, + retryable: !terminal, + terminal, + } +} + function validatePutAdminCertRequest(req: PutAdminCertRequest, shardus: Shardus): PutAdminCertResult { const publicKey = shardus.crypto.getPublicKey() @@ -79,29 +113,42 @@ export async function putAdminCertificateHandler(req: Request, shardus: Shardus) return { success: true } } -export async function tryAndFetchGoldenTicket(publicKey: string, network: NetworkAccount, dapp: Shardus): Promise { +export async function tryAndFetchGoldenTicket( + publicKey: string, + network: NetworkAccount, + dapp: Shardus, + isRetry = false, +): Promise { try { if (LiberdusFlags.VerboseLogs) console.log('Fetching golden ticket from', network.current.goldenTicketServerUrl, 'for publicKey', publicKey, 'node') const goldenTicketRequest: any = { publicKey, ip: config.server.ip.externalIp, port: config.server.ip.externalPort, - timestamp: dapp.shardusGetTime(), + timestamp: dapp.shardusGetTime() + (isRetry ? 1000 : 0), nonce: Math.floor(Math.random() * 1e6), } const signedGoldenTicketRequest: GoldenTicketRequest = dapp.signAsNode(goldenTicketRequest) if (LiberdusFlags.VerboseLogs) console.log('Golden Ticket request', signedGoldenTicketRequest) const response = await shardusPost(network.current.goldenTicketServerUrl, signedGoldenTicketRequest, { timeout: 5000 }) if (LiberdusFlags.VerboseLogs) console.log('Golden Ticket response', response.data) - if (response.data && response.data.success) { - return response.data.ticket + if (response.data && response.data.success && response.data.ticket) { + return { + ticket: response.data.ticket, + retryable: false, + terminal: false, + } } else { - console.error('No golden ticket received') - return null + const error = response.data?.error || 'No golden ticket received' + console.error(error) + return createGoldenTicketFetchResult(error) } } catch (error) { - console.error(`Error fetching golden ticket from - ${(error as Error).message}`) - return null + const axiosError = error as { message?: string; response?: { data?: AdminCertResponse; status?: number } } + const responseError = axiosError.response?.data?.error + const errorMessage = responseError || axiosError.message || 'Unknown Golden Ticket fetch error' + console.error(`Error fetching golden ticket from - ${errorMessage}`) + return createGoldenTicketFetchResult(errorMessage) } } export function setAdminCertificate(cert: AdminCert): void { diff --git a/src/transactions/apply_change_network_param.ts b/src/transactions/apply_change_network_param.ts index d5888d11..4a588262 100644 --- a/src/transactions/apply_change_network_param.ts +++ b/src/transactions/apply_change_network_param.ts @@ -79,6 +79,9 @@ export function backfillNetworkAccount(network: NetworkAccount): void { if (network.current.goldenTicketServerUrl === undefined || network.current.goldenTicketServerUrl === null) { network.current.goldenTicketServerUrl = 'http://localhost:3456/golden/ticket' } + if (network.current.goldenTicketRetryInterval === undefined || network.current.goldenTicketRetryInterval === null) { + network.current.goldenTicketRetryInterval = 10 * config.ONE_MINUTE + } if (network.current.messageRetentionDays === undefined || network.current.messageRetentionDays === null) { network.current.messageRetentionDays = 7 } diff --git a/test/admin_certificate.test.ts b/test/admin_certificate.test.ts new file mode 100644 index 00000000..fcc4b0d9 --- /dev/null +++ b/test/admin_certificate.test.ts @@ -0,0 +1,116 @@ +jest.mock('../src/utils/request', () => ({ + shardusPost: jest.fn(), +})) + +import { Shardus } from '@shardus/core' +import { shardusPost } from '../src/utils/request' +import { AdminCert, isTerminalGoldenTicketError, tryAndFetchGoldenTicket } from '../src/transactions/admin_certificate' +import { NetworkAccount } from '../src/@types' + +const mockShardusPost = shardusPost as jest.Mock + +const network = { + current: { + goldenTicketServerUrl: 'http://localhost:3456/golden/ticket', + }, +} as NetworkAccount + +const dapp = { + shardusGetTime: jest.fn(() => 123456), + signAsNode: jest.fn((request) => ({ ...request, sign: { owner: request.publicKey, sig: 'signature' } })), +} as unknown as Shardus + +const ticket: AdminCert = { + nominee: 'public-key', + certCreation: 123456, + certExp: 223456, + goldenTicket: true, + sign: { + owner: 'server', + sig: 'signature', + }, +} + +describe('Golden Ticket fetch', () => { + beforeEach(() => { + mockShardusPost.mockReset() + jest.spyOn(console, 'error').mockImplementation(() => undefined) + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + + it('returns a ticket on a successful Golden Ticket response', async () => { + mockShardusPost.mockResolvedValue({ data: { success: true, ticket } }) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result).toEqual({ ticket, retryable: false, terminal: false }) + }) + + it('classifies timestamp errors from HTTP responses as retryable', async () => { + mockShardusPost.mockRejectedValue({ + message: 'Request failed with status code 409', + response: { + status: 409, + data: { + success: false, + error: 'Timestamp out of acceptable range. Difference: 10000ms, Tolerance: 5000ms', + }, + }, + }) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result.retryable).toBe(true) + expect(result.terminal).toBe(false) + expect(result.error).toContain('Timestamp out of acceptable range') + }) + + it('classifies unregistered public keys as terminal', async () => { + mockShardusPost.mockRejectedValue({ + message: 'Request failed with status code 401', + response: { + status: 401, + data: { + success: false, + error: 'Public key not registered or inactive', + }, + }, + }) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result.retryable).toBe(false) + expect(result.terminal).toBe(true) + expect(result.error).toBe('Public key not registered or inactive') + }) + + it('classifies network failures as retryable', async () => { + mockShardusPost.mockRejectedValue(new Error('connect ECONNREFUSED 127.0.0.1:3456')) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result.retryable).toBe(true) + expect(result.terminal).toBe(false) + expect(result.error).toContain('ECONNREFUSED') + }) + + it('adds one second to the timestamp when retrying', async () => { + mockShardusPost.mockResolvedValue({ data: { success: true, ticket } }) + + await tryAndFetchGoldenTicket('public-key', network, dapp, true) + + expect(dapp.signAsNode).toHaveBeenCalledWith(expect.objectContaining({ timestamp: 124456 })) + }) + + it('identifies terminal Golden Ticket validation errors', () => { + expect(isTerminalGoldenTicketError('Public key not registered or inactive')).toBe(true) + expect(isTerminalGoldenTicketError('Schema validation failed: publicKey must be a string')).toBe(true) + expect(isTerminalGoldenTicketError('Nonce must be a non-negative integer')).toBe(true) + expect(isTerminalGoldenTicketError('Port must be between 1 and 65535')).toBe(true) + expect(isTerminalGoldenTicketError('Timestamp out of acceptable range')).toBe(false) + expect(isTerminalGoldenTicketError('Rate limit exceeded for this validator')).toBe(false) + }) +})