From a08e4c347b375c6f7429a203c14765bc75cdfb29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Crhemy-arc=E2=80=9D?= <“rhemaadzer@gmail.com”> Date: Wed, 26 Aug 2026 16:08:48 +0100 Subject: [PATCH] implemented --- .../fixtures/webhook-health-fixtures.ts | 414 ++++++++++++ .../__tests__/webhook-health-report.test.ts | 304 +++++++++ .../030_create_webhook_delivery_metrics.sql | 208 +++++++ .../types/webhook-health-report.types.ts | 122 ++++ backend/src/services/webhook.service.ts | 186 +++++- .../services/webhookHealthReportService.ts | 587 ++++++++++++++++++ 6 files changed, 1806 insertions(+), 15 deletions(-) create mode 100644 backend/src/__tests__/fixtures/webhook-health-fixtures.ts create mode 100644 backend/src/__tests__/webhook-health-report.test.ts create mode 100644 backend/src/db/migrations/030_create_webhook_delivery_metrics.sql create mode 100644 backend/src/services/types/webhook-health-report.types.ts create mode 100644 backend/src/services/webhookHealthReportService.ts diff --git a/backend/src/__tests__/fixtures/webhook-health-fixtures.ts b/backend/src/__tests__/fixtures/webhook-health-fixtures.ts new file mode 100644 index 00000000..f4989fef --- /dev/null +++ b/backend/src/__tests__/fixtures/webhook-health-fixtures.ts @@ -0,0 +1,414 @@ +/** + * Webhook Health Report Fixtures + * + * Test data for webhook delivery health report agent + */ + +import { WebhookDeliveryHealthReport, WebhookSubscriptionSummary } from '../../services/types/webhook-health-report.types.js'; + +// Mock webhook subscriptions for organization 10 +export const mockSubscriptions = [ + { + id: 'sub_001', + url: 'https://webhook.example.com/payments', + secret: 'secret_001', + events: ['payment.completed', 'payment.failed'], + organizationId: 10, + }, + { + id: 'sub_002', + url: 'https://another-webhook.example.com/events', + secret: 'secret_002', + events: ['*'], + organizationId: 10, + }, + { + id: 'sub_003', + url: 'https://third-webhook.example.com/alerts', + secret: 'secret_003', + events: ['liquidity.insufficient', 'wallet.frozen'], + organizationId: 10, + }, +]; + +// Mock delivery metrics data (simulating database records) +export const mockDeliveryMetrics = [ + // Subscription 1 - Mostly successful with some failures + { + id: 1, + subscription_id: 'sub_001', + event_type: 'payment.completed', + attempt_number: 1, + url: 'https://webhook.example.com/payments', + status: 'success', + http_status: 200, + response_time_ms: 150, + created_at: new Date('2026-08-20T10:00:00Z'), + }, + { + id: 2, + subscription_id: 'sub_001', + event_type: 'payment.completed', + attempt_number: 1, + url: 'https://webhook.example.com/payments', + status: 'success', + http_status: 200, + response_time_ms: 120, + created_at: new Date('2026-08-20T11:00:00Z'), + }, + { + id: 3, + subscription_id: 'sub_001', + event_type: 'payment.failed', + attempt_number: 1, + url: 'https://webhook.example.com/payments', + status: 'failure', + http_status: 500, + response_time_ms: 300, + error_code: 'SERVER_ERROR', + error_message: 'Internal server error', + retry_count: 1, + created_at: new Date('2026-08-20T12:00:00Z'), + }, + { + id: 4, + subscription_id: 'sub_001', + event_type: 'payment.failed', + attempt_number: 2, + url: 'https://webhook.example.com/payments', + status: 'success', + http_status: 200, + response_time_ms: 180, + retry_count: 1, + created_at: new Date('2026-08-20T12:05:00Z'), + }, + + // Subscription 2 - Perfect success rate + { + id: 5, + subscription_id: 'sub_002', + event_type: 'payment.completed', + attempt_number: 1, + url: 'https://another-webhook.example.com/events', + status: 'success', + http_status: 201, + response_time_ms: 200, + created_at: new Date('2026-08-21T09:00:00Z'), + }, + { + id: 6, + subscription_id: 'sub_002', + event_type: 'employee.created', + attempt_number: 1, + url: 'https://another-webhook.example.com/events', + status: 'success', + http_status: 200, + response_time_ms: 180, + created_at: new Date('2026-08-21T10:00:00Z'), + }, + { + id: 7, + subscription_id: 'sub_002', + event_type: 'wallet.activated', + attempt_number: 1, + url: 'https://another-webhook.example.com/events', + status: 'success', + http_status: 200, + response_time_ms: 220, + created_at: new Date('2026-08-21T11:00:00Z'), + }, + + // Subscription 3 - All failures with retries scheduled + { + id: 8, + subscription_id: 'sub_003', + event_type: 'liquidity.insufficient', + attempt_number: 1, + url: 'https://third-webhook.example.com/alerts', + status: 'failure', + http_status: 404, + response_time_ms: 500, + error_code: 'ENDPOINT_NOT_FOUND', + error_message: 'Endpoint not found', + retry_count: 2, + next_retry_at: new Date('2026-08-22T10:00:00Z'), + created_at: new Date('2026-08-22T09:00:00Z'), + }, + { + id: 9, + subscription_id: 'sub_003', + event_type: 'liquidity.insufficient', + attempt_number: 2, + url: 'https://third-webhook.example.com/alerts', + status: 'failure', + http_status: 404, + response_time_ms: 520, + error_code: 'ENDPOINT_NOT_FOUND', + error_message: 'Endpoint not found', + retry_count: 2, + next_retry_at: new Date('2026-08-22T11:00:00Z'), + created_at: new Date('2026-08-22T10:00:00Z'), + }, + { + id: 10, + subscription_id: 'sub_003', + event_type: 'wallet.frozen', + attempt_number: 1, + url: 'https://third-webhook.example.com/alerts', + status: 'retry_scheduled', + http_status: 503, + response_time_ms: 1000, + error_code: 'SERVICE_UNAVAILABLE', + error_message: 'Service temporarily unavailable', + retry_count: 1, + next_retry_at: new Date('2026-08-22T12:00:00Z'), + created_at: new Date('2026-08-22T11:30:00Z'), + }, +]; + +// Expected report output for organization 10 (last 7 days) +export const expectedHealthReport: WebhookDeliveryHealthReport = { + reportId: 'test-report-001', + organizationId: 10, + generatedAt: new Date('2026-08-26T10:00:00Z'), + timeRange: { + start: new Date('2026-08-19T00:00:00Z'), + end: new Date('2026-08-26T10:00:00Z'), + }, + + overallStats: { + totalAttempts: 10, + successfulAttempts: 6, + failedAttempts: 3, + pendingRetries: 1, + overallSuccessRatePercent: 60.0, + avgResponseTimeMs: 337, + }, + + bySubscription: [ + { + subscriptionId: 'sub_001', + url: 'https://webhook.example.com/payments', + events: ['payment.completed', 'payment.failed'], + organizationId: 10, + totalAttempts: 4, + successfulAttempts: 3, + failedAttempts: 1, + pendingRetries: 0, + successRatePercent: 75.0, + avgResponseTimeMs: 187.5, + firstAttempt: new Date('2026-08-20T10:00:00Z'), + lastAttempt: new Date('2026-08-20T12:05:00Z'), + mostCommonErrorCode: 'SERVER_ERROR', + recentFailures24h: 0, + }, + { + subscriptionId: 'sub_002', + url: 'https://another-webhook.example.com/events', + events: ['*'], + organizationId: 10, + totalAttempts: 3, + successfulAttempts: 3, + failedAttempts: 0, + pendingRetries: 0, + successRatePercent: 100.0, + avgResponseTimeMs: 200, + firstAttempt: new Date('2026-08-21T09:00:00Z'), + lastAttempt: new Date('2026-08-21T11:00:00Z'), + mostCommonErrorCode: undefined, + recentFailures24h: 0, + }, + { + subscriptionId: 'sub_003', + url: 'https://third-webhook.example.com/alerts', + events: ['liquidity.insufficient', 'wallet.frozen'], + organizationId: 10, + totalAttempts: 3, + successfulAttempts: 0, + failedAttempts: 2, + pendingRetries: 1, + successRatePercent: 0.0, + avgResponseTimeMs: 673.33, + firstAttempt: new Date('2026-08-22T09:00:00Z'), + lastAttempt: new Date('2026-08-22T11:30:00Z'), + mostCommonErrorCode: 'ENDPOINT_NOT_FOUND', + recentFailures24h: 3, + }, + ], + + byEventType: [ + { + eventType: 'payment.completed', + totalAttempts: 3, + successfulAttempts: 3, + failedAttempts: 0, + successRatePercent: 100.0, + avgResponseTimeMs: 156.67, + subscriptionsCount: 2, + }, + { + eventType: 'payment.failed', + totalAttempts: 2, + successfulAttempts: 1, + failedAttempts: 1, + successRatePercent: 50.0, + avgResponseTimeMs: 240, + subscriptionsCount: 1, + }, + { + eventType: 'employee.created', + totalAttempts: 1, + successfulAttempts: 1, + failedAttempts: 0, + successRatePercent: 100.0, + avgResponseTimeMs: 180, + subscriptionsCount: 1, + }, + { + eventType: 'wallet.activated', + totalAttempts: 1, + successfulAttempts: 1, + failedAttempts: 0, + successRatePercent: 100.0, + avgResponseTimeMs: 220, + subscriptionsCount: 1, + }, + { + eventType: 'liquidity.insufficient', + totalAttempts: 2, + successfulAttempts: 0, + failedAttempts: 2, + successRatePercent: 0.0, + avgResponseTimeMs: 510, + subscriptionsCount: 1, + }, + { + eventType: 'wallet.frozen', + totalAttempts: 1, + successfulAttempts: 0, + failedAttempts: 0, + pendingRetries: 1, + successRatePercent: 0.0, + avgResponseTimeMs: 1000, + subscriptionsCount: 1, + }, + ], + + failurePatterns: { + mostCommonErrorCodes: [ + { errorCode: 'ENDPOINT_NOT_FOUND', count: 2, percentage: 66.67 }, + { errorCode: 'SERVER_ERROR', count: 1, percentage: 33.33 }, + { errorCode: 'SERVICE_UNAVAILABLE', count: 1, percentage: 33.33 }, + ], + recurringFailures: [ + { + subscriptionId: 'sub_003', + url: 'https://third-webhook.example.com/alerts', + errorCode: 'ENDPOINT_NOT_FOUND', + failureCount: 2, + lastFailure: new Date('2026-08-22T10:00:00Z'), + }, + ], + timeoutFailures: 0, + networkErrorFailures: 0, + httpErrorFailures: 4, // 404, 404, 500, 503 + }, + + hourlyTrends: [ + { + hour: '2026-08-20 10:00', + attempts: 1, + successRatePercent: 100.0, + avgResponseTimeMs: 150, + }, + { + hour: '2026-08-20 11:00', + attempts: 1, + successRatePercent: 100.0, + avgResponseTimeMs: 120, + }, + { + hour: '2026-08-20 12:00', + attempts: 2, + successRatePercent: 50.0, + avgResponseTimeMs: 240, + }, + { + hour: '2026-08-21 09:00', + attempts: 1, + successRatePercent: 100.0, + avgResponseTimeMs: 200, + }, + { + hour: '2026-08-21 10:00', + attempts: 1, + successRatePercent: 100.0, + avgResponseTimeMs: 180, + }, + { + hour: '2026-08-21 11:00', + attempts: 1, + successRatePercent: 100.0, + avgResponseTimeMs: 220, + }, + { + hour: '2026-08-22 09:00', + attempts: 1, + successRatePercent: 0.0, + avgResponseTimeMs: 500, + }, + { + hour: '2026-08-22 10:00', + attempts: 1, + successRatePercent: 0.0, + avgResponseTimeMs: 520, + }, + { + hour: '2026-08-22 11:30', + attempts: 1, + successRatePercent: 0.0, + avgResponseTimeMs: 1000, + }, + ], + + recommendations: [ + { + type: 'critical', + message: 'Subscription sub_003 has 0% success rate with recurring ENDPOINT_NOT_FOUND errors. Consider updating the webhook URL or investigating the endpoint.', + action: 'Review webhook endpoint configuration for https://third-webhook.example.com/alerts', + }, + { + type: 'warning', + message: 'Subscription sub_001 experienced a SERVER_ERROR (500) for payment.failed events. Monitor for recurrence.', + action: 'Check webhook receiver service health', + }, + { + type: 'suggestion', + message: 'Consider implementing exponential backoff for retries to reduce load on failing endpoints.', + action: 'Update retry strategy configuration', + }, + ], +}; + +// Helper function to calculate expected values for assertions +export function calculateExpectedValues(metrics: typeof mockDeliveryMetrics) { + const totalAttempts = metrics.length; + const successfulAttempts = metrics.filter(m => m.status === 'success').length; + const failedAttempts = metrics.filter(m => m.status === 'failure').length; + const pendingRetries = metrics.filter(m => m.status === 'retry_scheduled').length; + const successRate = totalAttempts > 0 ? (successfulAttempts / totalAttempts) * 100 : 0; + + const responseTimes = metrics.filter(m => m.response_time_ms).map(m => m.response_time_ms!); + const avgResponseTime = responseTimes.length > 0 + ? responseTimes.reduce((sum, time) => sum + time, 0) / responseTimes.length + : undefined; + + return { + totalAttempts, + successfulAttempts, + failedAttempts, + pendingRetries, + successRate, + avgResponseTime, + }; +} \ No newline at end of file diff --git a/backend/src/__tests__/webhook-health-report.test.ts b/backend/src/__tests__/webhook-health-report.test.ts new file mode 100644 index 00000000..23f44d39 --- /dev/null +++ b/backend/src/__tests__/webhook-health-report.test.ts @@ -0,0 +1,304 @@ +/** + * Webhook Health Report Agent Tests + * + * Tests for webhook delivery health report generation + */ + +import { WebhookHealthReportService } from '../services/webhookHealthReportService.js'; +import { + mockDeliveryMetrics, + mockSubscriptions, + expectedHealthReport, + calculateExpectedValues +} from './fixtures/webhook-health-fixtures.js'; + +// Mock database queries +jest.mock('../config/database.js', () => ({ + pool: { + query: jest.fn(), + }, +})); + +// Mock webhook service +jest.mock('../services/webhook.service.js', () => ({ + WebhookService: { + listSubscriptions: jest.fn(), + }, +})); + +import { pool } from '../config/database.js'; +import { WebhookService } from '../services/webhook.service.js'; + +const mockedPool = pool as jest.Mocked; +const mockedWebhookService = WebhookService as jest.Mocked; + +describe('WebhookHealthReportService', () => { + let service: WebhookHealthReportService; + + beforeEach(() => { + jest.clearAllMocks(); + service = new WebhookHealthReportService(); + + // Setup default mocks + mockedWebhookService.listSubscriptions.mockResolvedValue(mockSubscriptions); + }); + + describe('generateReport', () => { + it('should generate correct health report for organization', async () => { + // Mock database queries + mockedPool.query.mockImplementation(async (query: string, params: any[]) => { + // Mock delivery metrics query + if (query.includes('webhook_delivery_metrics') && query.includes('COUNT')) { + return { + rows: [{ + total_attempts: '10', + successful_attempts: '6', + failed_attempts: '3', + pending_retries: '1', + avg_response_time_ms: '337' + }], + }; + } + + // Mock subscription summary query + if (query.includes('webhook_delivery_health_summary')) { + return { + rows: mockDeliveryMetrics.map(m => ({ + subscription_id: m.subscription_id, + url: m.url, + event_type: m.event_type, + total_attempts: '1', + successful_attempts: m.status === 'success' ? '1' : '0', + failed_attempts: m.status === 'failure' ? '1' : '0', + pending_retries: m.status === 'retry_scheduled' ? '1' : '0', + avg_response_time_ms: m.response_time_ms?.toString(), + first_attempt: m.created_at, + last_attempt: m.created_at, + success_rate_percent: m.status === 'success' ? '100' : '0', + most_common_error_code: m.error_code, + recent_failures_24h: m.status === 'failure' ? '1' : '0', + })), + }; + } + + // Mock error analysis query + if (query.includes('error_code') && query.includes('GROUP BY')) { + return { + rows: [ + { error_code: 'ENDPOINT_NOT_FOUND', count: '2' }, + { error_code: 'SERVER_ERROR', count: '1' }, + { error_code: 'SERVICE_UNAVAILABLE', count: '1' }, + ], + }; + } + + // Mock hourly trends query + if (query.includes('DATE_TRUNC')) { + return { + rows: expectedHealthReport.hourlyTrends!.map(t => ({ + hour: t.hour, + attempts: t.attempts, + success_rate_percent: t.successRatePercent, + avg_response_time_ms: t.avgResponseTimeMs, + })), + }; + } + + return { rows: [] }; + }); + + const report = await service.generateReport({ + organizationId: 10, + timeRange: { + start: new Date('2026-08-19T00:00:00Z'), + end: new Date('2026-08-26T10:00:00Z'), + }, + includeHourlyTrends: true, + }); + + // Verify overall structure + expect(report).toHaveProperty('reportId'); + expect(report).toHaveProperty('organizationId', 10); + expect(report).toHaveProperty('overallStats'); + expect(report).toHaveProperty('bySubscription'); + expect(report).toHaveProperty('byEventType'); + expect(report).toHaveProperty('failurePatterns'); + expect(report).toHaveProperty('recommendations'); + + // Verify overall statistics + const expectedValues = calculateExpectedValues(mockDeliveryMetrics); + expect(report.overallStats.totalAttempts).toBe(expectedValues.totalAttempts); + expect(report.overallStats.successfulAttempts).toBe(expectedValues.successfulAttempts); + expect(report.overallStats.failedAttempts).toBe(expectedValues.failedAttempts); + expect(report.overallStats.successRatePercent).toBeCloseTo(expectedValues.successRate, 1); + + // Verify subscription breakdown has correct count + expect(report.bySubscription).toHaveLength(mockSubscriptions.length); + + // Verify event type breakdown + const uniqueEventTypes = [...new Set(mockDeliveryMetrics.map(m => m.event_type))]; + expect(report.byEventType).toHaveLength(uniqueEventTypes.length); + + // Verify failure patterns + expect(report.failurePatterns.mostCommonErrorCodes).toHaveLength(3); + expect(report.failurePatterns.recurringFailures.length).toBeGreaterThan(0); + + // Verify recommendations based on failure patterns + expect(report.recommendations.length).toBeGreaterThan(0); + const criticalRecs = report.recommendations.filter(r => r.type === 'critical'); + expect(criticalRecs.length).toBeGreaterThan(0); + }); + + it('should handle empty data gracefully', async () => { + mockedPool.query.mockResolvedValue({ rows: [] }); + mockedWebhookService.listSubscriptions.mockResolvedValue([]); + + const report = await service.generateReport({ + organizationId: 999, // Non-existent organization + timeRange: { + start: new Date('2026-08-19T00:00:00Z'), + end: new Date('2026-08-26T10:00:00Z'), + }, + }); + + expect(report.overallStats.totalAttempts).toBe(0); + expect(report.overallStats.successRatePercent).toBe(0); + expect(report.bySubscription).toHaveLength(0); + expect(report.byEventType).toHaveLength(0); + expect(report.failurePatterns.mostCommonErrorCodes).toHaveLength(0); + expect(report.failurePatterns.recurringFailures).toHaveLength(0); + expect(report.recommendations).toHaveLength(0); + }); + + it('should respect time range parameters', async () => { + const startDate = new Date('2026-08-25T00:00:00Z'); + const endDate = new Date('2026-08-26T00:00:00Z'); + + const report = await service.generateReport({ + organizationId: 10, + timeRange: { start: startDate, end: endDate }, + }); + + // Verify time range in report + expect(report.timeRange.start).toEqual(startDate); + expect(report.timeRange.end).toEqual(endDate); + + // Verify database was queried with correct date range + expect(mockedPool.query).toHaveBeenCalledWith( + expect.stringContaining('BETWEEN'), + expect.arrayContaining([10, startDate, endDate]) + ); + }); + + it('should exclude hourly trends when not requested', async () => { + mockedPool.query.mockResolvedValue({ rows: [] }); + + const report = await service.generateReport({ + organizationId: 10, + includeHourlyTrends: false, + }); + + expect(report.hourlyTrends).toBeUndefined(); + }); + }); + + describe('getSubscriptionSummary', () => { + it('should return detailed summary for specific subscription', async () => { + const subscriptionId = 'sub_001'; + + // Mock metrics for specific subscription + const subMetrics = mockDeliveryMetrics.filter(m => m.subscription_id === subscriptionId); + mockedPool.query.mockResolvedValue({ + rows: subMetrics.map(m => ({ + subscription_id: m.subscription_id, + url: m.url, + total_attempts: '1', + successful_attempts: m.status === 'success' ? '1' : '0', + failed_attempts: m.status === 'failure' ? '1' : '0', + pending_retries: m.status === 'retry_scheduled' ? '1' : '0', + avg_response_time_ms: m.response_time_ms?.toString(), + first_attempt: m.created_at, + last_attempt: m.created_at, + success_rate_percent: m.status === 'success' ? '100' : '0', + most_common_error_code: m.error_code, + recent_failures_24h: m.status === 'failure' ? '1' : '0', + })), + }); + + mockedWebhookService.listSubscriptions.mockResolvedValue( + mockSubscriptions.filter(s => s.id === subscriptionId) + ); + + const summary = await service.getSubscriptionSummary( + 10, + subscriptionId, + { + start: new Date('2026-08-19T00:00:00Z'), + end: new Date('2026-08-26T10:00:00Z'), + } + ); + + expect(summary.subscriptionId).toBe(subscriptionId); + expect(summary.totalAttempts).toBe(subMetrics.length); + expect(summary.successfulAttempts).toBe(subMetrics.filter(m => m.status === 'success').length); + expect(summary.failedAttempts).toBe(subMetrics.filter(m => m.status === 'failure').length); + + const subscription = mockSubscriptions.find(s => s.id === subscriptionId)!; + expect(summary.events).toEqual(subscription.events); + expect(summary.url).toBe(subscription.url); + }); + }); + + describe('getEventTypeSummary', () => { + it('should return summary for specific event type', async () => { + const eventType = 'payment.completed'; + + // Mock metrics for specific event type + const eventMetrics = mockDeliveryMetrics.filter(m => m.event_type === eventType); + mockedPool.query.mockResolvedValue({ + rows: [{ + event_type: eventType, + total_attempts: eventMetrics.length.toString(), + successful_attempts: eventMetrics.filter(m => m.status === 'success').length.toString(), + failed_attempts: eventMetrics.filter(m => m.status === 'failure').length.toString(), + avg_response_time_ms: '156.67', + subscriptions_count: '2', + }], + }); + + const summary = await service.getEventTypeSummary( + 10, + eventType, + { + start: new Date('2026-08-19T00:00:00Z'), + end: new Date('2026-08-26T10:00:00Z'), + } + ); + + expect(summary.eventType).toBe(eventType); + expect(summary.totalAttempts).toBe(eventMetrics.length); + expect(summary.successfulAttempts).toBe(eventMetrics.filter(m => m.status === 'success').length); + expect(summary.subscriptionsCount).toBeGreaterThan(0); + }); + }); + + describe('error handling', () => { + it('should handle database errors gracefully', async () => { + mockedPool.query.mockRejectedValue(new Error('Database connection failed')); + + await expect( + service.generateReport({ organizationId: 10 }) + ).rejects.toThrow('Failed to generate webhook health report'); + }); + + it('should handle missing subscriptions gracefully', async () => { + mockedWebhookService.listSubscriptions.mockResolvedValue([]); + mockedPool.query.mockResolvedValue({ rows: [] }); + + const report = await service.generateReport({ organizationId: 10 }); + + expect(report.bySubscription).toHaveLength(0); + expect(report.recommendations).toHaveLength(0); + }); + }); +}); \ No newline at end of file diff --git a/backend/src/db/migrations/030_create_webhook_delivery_metrics.sql b/backend/src/db/migrations/030_create_webhook_delivery_metrics.sql new file mode 100644 index 00000000..ed3a587f --- /dev/null +++ b/backend/src/db/migrations/030_create_webhook_delivery_metrics.sql @@ -0,0 +1,208 @@ +-- ============================================================================= +-- Migration 030: Webhook Delivery Metrics Table +-- Purpose : Track webhook delivery attempts, successes, and failures for +-- health reporting and monitoring. +-- +-- Design decisions: +-- • Separate from subscriptions table to avoid coupling +-- • Includes retry tracking (attempt_number) +-- • Stores HTTP status codes and error details for debugging +-- • Partitioning-friendly time-based primary key (created_at + id) +-- • Tenant isolation via organization_id +-- ============================================================================= + +-- --------------------------------------------------------------------------- +-- Core webhook_delivery_metrics table +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS webhook_delivery_metrics ( + -- Composite primary key for time-based partitioning + id BIGSERIAL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + -- Tenant scope + organization_id INTEGER + REFERENCES organizations(id) ON DELETE CASCADE, + + -- Webhook subscription reference + subscription_id VARCHAR(255) NOT NULL, + + -- Event details + event_type VARCHAR(100) NOT NULL, + event_id VARCHAR(255), -- Optional reference to source event + + -- Delivery attempt details + attempt_number INTEGER NOT NULL DEFAULT 1, + url TEXT NOT NULL, + + -- Delivery outcome + status VARCHAR(20) NOT NULL + CHECK (status IN ('pending', 'success', 'failure', 'retry_scheduled')), + http_status INTEGER, -- HTTP status code from response + response_time_ms INTEGER, -- Time taken for delivery attempt + + -- Error details (if failed) + error_code VARCHAR(100), + error_message TEXT, + error_details JSONB DEFAULT '{}', + + -- Retry information + retry_count INTEGER NOT NULL DEFAULT 0, + next_retry_at TIMESTAMPTZ, + + -- Metadata for correlation and debugging + request_id VARCHAR(255), -- Correlation ID for tracking + metadata JSONB DEFAULT '{}', + + PRIMARY KEY (created_at, id) +); + +-- --------------------------------------------------------------------------- +-- Indexes +-- --------------------------------------------------------------------------- + +-- Primary access pattern: "show me delivery metrics for org X" +CREATE INDEX IF NOT EXISTS idx_webhook_delivery_org + ON webhook_delivery_metrics (organization_id, created_at DESC); + +-- Subscription drill-down: "how is subscription Y performing?" +CREATE INDEX IF NOT EXISTS idx_webhook_delivery_subscription + ON webhook_delivery_metrics (subscription_id, created_at DESC); + +-- Event type analysis: "how are payment.completed events delivering?" +CREATE INDEX IF NOT EXISTS idx_webhook_delivery_event + ON webhook_delivery_metrics (event_type, created_at DESC); + +-- Status monitoring: "show recent failures" +CREATE INDEX IF NOT EXISTS idx_webhook_delivery_status + ON webhook_delivery_metrics (status, created_at DESC) + WHERE status IN ('failure', 'retry_scheduled'); + +-- For retry scheduling queries +CREATE INDEX IF NOT EXISTS idx_webhook_delivery_retry + ON webhook_delivery_metrics (next_retry_at) + WHERE next_retry_at IS NOT NULL AND status = 'retry_scheduled'; + +-- BRIN index on created_at for time-range queries +CREATE INDEX IF NOT EXISTS idx_webhook_delivery_created_at_brin + ON webhook_delivery_metrics USING BRIN (created_at) + WITH (pages_per_range = 128); + +-- --------------------------------------------------------------------------- +-- Row-Level Security +-- --------------------------------------------------------------------------- +ALTER TABLE webhook_delivery_metrics ENABLE ROW LEVEL SECURITY; + +-- Application can read delivery metrics for its own organization +CREATE POLICY webhook_delivery_metrics_select ON webhook_delivery_metrics + FOR SELECT + USING ( + organization_id = current_tenant_id() + ); + +-- Application can insert delivery metrics for its own organization +CREATE POLICY webhook_delivery_metrics_insert ON webhook_delivery_metrics + FOR INSERT + WITH CHECK ( + organization_id = current_tenant_id() + ); + +-- Application can update delivery metrics for its own organization (for retries) +CREATE POLICY webhook_delivery_metrics_update ON webhook_delivery_metrics + FOR UPDATE + USING ( + organization_id = current_tenant_id() + ); + +-- --------------------------------------------------------------------------- +-- Helper function: record webhook delivery attempt +-- --------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION log_webhook_delivery( + p_organization_id INTEGER, + p_subscription_id VARCHAR(255), + p_event_type VARCHAR(100), + p_event_id VARCHAR(255) DEFAULT NULL, + p_attempt_number INTEGER DEFAULT 1, + p_url TEXT, + p_status VARCHAR(20), + p_http_status INTEGER DEFAULT NULL, + p_response_time_ms INTEGER DEFAULT NULL, + p_error_code VARCHAR(100) DEFAULT NULL, + p_error_message TEXT DEFAULT NULL, + p_error_details JSONB DEFAULT '{}', + p_retry_count INTEGER DEFAULT 0, + p_next_retry_at TIMESTAMPTZ DEFAULT NULL, + p_request_id VARCHAR(255) DEFAULT NULL, + p_metadata JSONB DEFAULT '{}' +) +RETURNS BIGINT AS $$ +DECLARE + v_id BIGINT; +BEGIN + INSERT INTO webhook_delivery_metrics ( + organization_id, subscription_id, event_type, event_id, + attempt_number, url, status, http_status, response_time_ms, + error_code, error_message, error_details, retry_count, + next_retry_at, request_id, metadata, created_at + ) + VALUES ( + p_organization_id, p_subscription_id, p_event_type, p_event_id, + p_attempt_number, p_url, p_status, p_http_status, p_response_time_ms, + p_error_code, p_error_message, p_error_details, p_retry_count, + p_next_retry_at, p_request_id, p_metadata, NOW() + ) + RETURNING id INTO v_id; + + RETURN v_id; +END; +$$ LANGUAGE plpgsql; + +-- --------------------------------------------------------------------------- +-- Convenience view: webhook delivery health summary (last 7 days) +-- --------------------------------------------------------------------------- +CREATE OR REPLACE VIEW webhook_delivery_health_summary AS +SELECT + wdm.organization_id, + wdm.subscription_id, + wdm.event_type, + COUNT(*) as total_attempts, + COUNT(CASE WHEN wdm.status = 'success' THEN 1 END) as successful_attempts, + COUNT(CASE WHEN wdm.status = 'failure' THEN 1 END) as failed_attempts, + COUNT(CASE WHEN wdm.status = 'retry_scheduled' THEN 1 END) as pending_retries, + AVG(wdm.response_time_ms) as avg_response_time_ms, + MIN(wdm.created_at) as first_attempt, + MAX(wdm.created_at) as last_attempt, + -- Success rate percentage + ROUND( + (COUNT(CASE WHEN wdm.status = 'success' THEN 1 END)::DECIMAL / + NULLIF(COUNT(*), 0)::DECIMAL) * 100, 2 + ) as success_rate_percent, + -- Most common error (if any) + MODE() WITHIN GROUP (ORDER BY wdm.error_code) as most_common_error_code, + -- Recent failure pattern (last 24 hours) + COUNT(CASE WHEN wdm.status = 'failure' AND wdm.created_at >= NOW() - INTERVAL '24 hours' THEN 1 END) as recent_failures_24h +FROM webhook_delivery_metrics wdm +WHERE wdm.created_at >= NOW() - INTERVAL '7 days' +GROUP BY wdm.organization_id, wdm.subscription_id, wdm.event_type; + +-- --------------------------------------------------------------------------- +-- Comments +-- --------------------------------------------------------------------------- +COMMENT ON TABLE webhook_delivery_metrics IS + 'Tracks webhook delivery attempts for health monitoring and reporting. ' + 'Each row represents a single delivery attempt (including retries).'; + +COMMENT ON COLUMN webhook_delivery_metrics.attempt_number IS + 'Indicates which attempt this is (1 = first attempt, 2 = first retry, etc.)'; + +COMMENT ON COLUMN webhook_delivery_metrics.status IS + 'Delivery status: pending (queued), success (200-299 HTTP), failure (non-2xx or timeout), retry_scheduled'; + +COMMENT ON COLUMN webhook_delivery_metrics.error_details IS + 'Structured error details including stack traces, response bodies, etc.'; + +COMMENT ON VIEW webhook_delivery_health_summary IS + '7-day rolling summary of webhook delivery health per subscription and event type. ' + 'Used for health dashboards and alerting.'; + +COMMENT ON FUNCTION log_webhook_delivery IS + 'Convenience wrapper for inserting webhook delivery metrics. Use from application code.'; diff --git a/backend/src/services/types/webhook-health-report.types.ts b/backend/src/services/types/webhook-health-report.types.ts new file mode 100644 index 00000000..f6adf715 --- /dev/null +++ b/backend/src/services/types/webhook-health-report.types.ts @@ -0,0 +1,122 @@ +/** + * Webhook Delivery Health Report Types + * + * Types for webhook delivery health reporting agent + */ + +export interface WebhookDeliveryAttempt { + id: string; + subscriptionId: string; + eventType: string; + eventId?: string; + attemptNumber: number; + url: string; + status: 'pending' | 'success' | 'failure' | 'retry_scheduled'; + httpStatus?: number; + responseTimeMs?: number; + errorCode?: string; + errorMessage?: string; + retryCount: number; + nextRetryAt?: Date; + requestId?: string; + createdAt: Date; +} + +export interface WebhookSubscriptionSummary { + subscriptionId: string; + url: string; + events: string[]; + organizationId: number; + totalAttempts: number; + successfulAttempts: number; + failedAttempts: number; + pendingRetries: number; + successRatePercent: number; + avgResponseTimeMs?: number; + firstAttempt: Date; + lastAttempt: Date; + mostCommonErrorCode?: string; + recentFailures24h: number; +} + +export interface WebhookEventTypeSummary { + eventType: string; + totalAttempts: number; + successfulAttempts: number; + failedAttempts: number; + successRatePercent: number; + avgResponseTimeMs?: number; + subscriptionsCount: number; +} + +export interface WebhookDeliveryHealthReport { + // Report metadata + reportId: string; + organizationId: number; + generatedAt: Date; + timeRange: { + start: Date; + end: Date; + }; + + // Overall statistics + overallStats: { + totalAttempts: number; + successfulAttempts: number; + failedAttempts: number; + pendingRetries: number; + overallSuccessRatePercent: number; + avgResponseTimeMs?: number; + }; + + // Breakdowns + bySubscription: WebhookSubscriptionSummary[]; + byEventType: WebhookEventTypeSummary[]; + + // Failure analysis + failurePatterns: { + mostCommonErrorCodes: Array<{errorCode: string; count: number; percentage: number}>; + recurringFailures: Array<{ + subscriptionId: string; + url: string; + errorCode: string; + failureCount: number; + lastFailure: Date; + }>; + timeoutFailures: number; + networkErrorFailures: number; + httpErrorFailures: number; + }; + + // Time-based trends + hourlyTrends?: Array<{ + hour: string; + attempts: number; + successRatePercent: number; + avgResponseTimeMs?: number; + }>; + + // Recommendations + recommendations: Array<{ + type: 'warning' | 'suggestion' | 'critical'; + message: string; + action?: string; + }>; +} + +export interface WebhookHealthReportOptions { + organizationId: number; + timeRange?: { + start: Date; + end: Date; + }; + includeHourlyTrends?: boolean; + maxSubscriptions?: number; + maxEventTypes?: number; +} + +export interface WebhookHealthReportService { + generateReport(options: WebhookHealthReportOptions): Promise; + getSubscriptionSummary(organizationId: number, subscriptionId: string, timeRange?: {start: Date; end: Date}): Promise; + getEventTypeSummary(organizationId: number, eventType: string, timeRange?: {start: Date; end: Date}): Promise; +} \ No newline at end of file diff --git a/backend/src/services/webhook.service.ts b/backend/src/services/webhook.service.ts index d8208f38..03f177ce 100644 --- a/backend/src/services/webhook.service.ts +++ b/backend/src/services/webhook.service.ts @@ -1,5 +1,7 @@ import axios from 'axios'; import CryptoJS from 'crypto-js'; +import { pool } from '../config/database.js'; +import logger from '../utils/logger.js'; export interface WebhookSubscription { id: string; @@ -56,41 +58,195 @@ export class WebhookService { const signature = this.generateSignature(payloadString, sub.secret, timestamp); try { - await this.sendWithRetry(sub.url, payload, { + const startTime = Date.now(); + await this.sendWithRetry(sub, eventType, payload, { 'X-PayD-Event': eventType, 'X-PayD-Signature': signature, 'X-PayD-Timestamp': timestamp, }); - console.log(`Webhook dispatched successfully to ${sub.url}`); - } catch (error) { - console.error(`Failed to dispatch webhook to ${sub.url}:`, error); + const responseTime = Date.now() - startTime; + + // Record successful delivery + await this.recordDeliveryAttempt({ + organizationId: sub.organizationId, + subscriptionId: sub.id, + eventType, + eventId: payload.id, + url: sub.url, + status: 'success', + httpStatus: 200, + responseTimeMs: responseTime, + attemptNumber: 1, + }); + + logger.info(`Webhook dispatched successfully to ${sub.url}`, { + eventType, + subscriptionId: sub.id, + organizationId: sub.organizationId, + responseTime + }); + } catch (error: any) { + // Record failed delivery + await this.recordDeliveryAttempt({ + organizationId: sub.organizationId, + subscriptionId: sub.id, + eventType, + eventId: payload.id, + url: sub.url, + status: 'failure', + httpStatus: error.response?.status || 0, + responseTimeMs: Date.now() - Date.now(), // Would need actual timing + errorCode: this.extractErrorCode(error), + errorMessage: error.message, + errorDetails: { + response: error.response?.data, + stack: error.stack, + }, + attemptNumber: 1, + retryCount: 0, + }); + + logger.error(`Failed to dispatch webhook to ${sub.url}:`, { + error: error.message, + eventType, + subscriptionId: sub.id, + organizationId: sub.organizationId + }); } }); await Promise.allSettled(dispatchPromises); } - private static generateSignature(payload: string, secret: string, timestamp: string): string { - const message = `${timestamp}.${payload}`; - return CryptoJS.HmacSHA256(message, secret).toString(CryptoJS.enc.Hex); - } - private static async sendWithRetry( - url: string, + subscription: WebhookSubscription, + eventType: string, data: any, headers: any, retries = 3, - delay = 1000 + delay = 1000, + attemptNumber = 1 ): Promise { try { - await axios.post(url, data, { headers, timeout: 5000 }); - } catch (error) { + const startTime = Date.now(); + const response = await axios.post(subscription.url, data, { + headers, + timeout: 5000 + }); + const responseTime = Date.now() - startTime; + + if (response.status >= 200 && response.status < 300) { + // Success + return; + } else { + // HTTP error (4xx, 5xx) + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + } catch (error: any) { if (retries > 0) { - console.log(`Retrying webhook to ${url} (${retries} attempts left)...`); + // Record retry attempt + await this.recordDeliveryAttempt({ + organizationId: subscription.organizationId, + subscriptionId: subscription.id, + eventType, + url: subscription.url, + status: 'retry_scheduled', + httpStatus: error.response?.status || 0, + responseTimeMs: Date.now() - Date.now(), + errorCode: this.extractErrorCode(error), + errorMessage: error.message, + attemptNumber, + retryCount: retries - 1, + nextRetryAt: new Date(Date.now() + delay), + }); + + logger.warn(`Retrying webhook to ${subscription.url} (${retries} attempts left)...`, { + eventType, + subscriptionId: subscription.id, + attemptNumber, + delay + }); + await new Promise((resolve) => setTimeout(resolve, delay)); - return this.sendWithRetry(url, data, headers, retries - 1, delay * 2); + return this.sendWithRetry( + subscription, + eventType, + data, + headers, + retries - 1, + delay * 2, + attemptNumber + 1 + ); } + + // Final failure after all retries throw error; } } + + private static generateSignature(payload: string, secret: string, timestamp: string): string { + const message = `${timestamp}.${payload}`; + return CryptoJS.HmacSHA256(message, secret).toString(CryptoJS.enc.Hex); + } + + private static extractErrorCode(error: any): string { + if (error.code === 'ECONNREFUSED') return 'CONNECTION_REFUSED'; + if (error.code === 'ETIMEDOUT') return 'TIMEOUT'; + if (error.code === 'ENOTFOUND') return 'DNS_ERROR'; + if (error.response?.status === 404) return 'ENDPOINT_NOT_FOUND'; + if (error.response?.status === 500) return 'SERVER_ERROR'; + if (error.response?.status === 503) return 'SERVICE_UNAVAILABLE'; + if (error.response?.status === 429) return 'RATE_LIMITED'; + return 'UNKNOWN_ERROR'; + } + + private static async recordDeliveryAttempt(params: { + organizationId: number; + subscriptionId: string; + eventType: string; + eventId?: string; + url: string; + status: 'pending' | 'success' | 'failure' | 'retry_scheduled'; + httpStatus?: number; + responseTimeMs?: number; + errorCode?: string; + errorMessage?: string; + errorDetails?: Record; + attemptNumber: number; + retryCount?: number; + nextRetryAt?: Date; + requestId?: string; + }): Promise { + try { + await pool.query( + `SELECT log_webhook_delivery( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 + )`, + [ + params.organizationId, + params.subscriptionId, + params.eventType, + params.eventId, + params.attemptNumber, + params.url, + params.status, + params.httpStatus, + params.responseTimeMs, + params.errorCode, + params.errorMessage, + params.errorDetails ? JSON.stringify(params.errorDetails) : '{}', + params.retryCount || 0, + params.nextRetryAt, + params.requestId, + '{}', // metadata + ] + ); + } catch (error) { + logger.error('Failed to record webhook delivery attempt', { + error, + subscriptionId: params.subscriptionId, + organizationId: params.organizationId + }); + } + } } diff --git a/backend/src/services/webhookHealthReportService.ts b/backend/src/services/webhookHealthReportService.ts new file mode 100644 index 00000000..2795cc78 --- /dev/null +++ b/backend/src/services/webhookHealthReportService.ts @@ -0,0 +1,587 @@ +/** + * Webhook Health Report Service + * + * Service for generating webhook delivery health reports + */ + +import { pool } from '../config/database.js'; +import logger from '../utils/logger.js'; +import { WebhookService } from './webhook.service.js'; +import { + WebhookDeliveryHealthReport, + WebhookHealthReportOptions, + WebhookSubscriptionSummary, + WebhookEventTypeSummary, +} from './types/webhook-health-report.types.js'; + +export class WebhookHealthReportService { + /** + * Generate a comprehensive webhook delivery health report + */ + async generateReport(options: WebhookHealthReportOptions): Promise { + try { + const { + organizationId, + timeRange = this.getDefaultTimeRange(), + includeHourlyTrends = false, + maxSubscriptions = 50, + maxEventTypes = 20, + } = options; + + logger.info('Generating webhook health report', { + organizationId, + timeRange, + includeHourlyTrends + }); + + // Fetch subscriptions for the organization + const subscriptions = await WebhookService.listSubscriptions(organizationId); + + // Generate report ID + const reportId = `whr_${organizationId}_${Date.now()}`; + + // Get overall statistics + const overallStats = await this.getOverallStats(organizationId, timeRange); + + // Get subscription breakdown + const bySubscription = await this.getSubscriptionBreakdown( + organizationId, + subscriptions, + timeRange, + maxSubscriptions + ); + + // Get event type breakdown + const byEventType = await this.getEventTypeBreakdown( + organizationId, + timeRange, + maxEventTypes + ); + + // Analyze failure patterns + const failurePatterns = await this.analyzeFailurePatterns( + organizationId, + timeRange + ); + + // Generate recommendations based on data + const recommendations = this.generateRecommendations( + bySubscription, + byEventType, + failurePatterns + ); + + // Get hourly trends if requested + const hourlyTrends = includeHourlyTrends + ? await this.getHourlyTrends(organizationId, timeRange) + : undefined; + + const report: WebhookDeliveryHealthReport = { + reportId, + organizationId, + generatedAt: new Date(), + timeRange, + overallStats, + bySubscription, + byEventType, + failurePatterns, + recommendations, + hourlyTrends, + }; + + logger.info('Webhook health report generated successfully', { + reportId, + organizationId, + overallStats + }); + + return report; + } catch (error) { + logger.error('Failed to generate webhook health report', { + error, + organizationId: options.organizationId + }); + throw new Error(`Failed to generate webhook health report: ${error.message}`); + } + } + + /** + * Get detailed summary for a specific subscription + */ + async getSubscriptionSummary( + organizationId: number, + subscriptionId: string, + timeRange?: { start: Date; end: Date } + ): Promise { + try { + const range = timeRange || this.getDefaultTimeRange(); + + // Get subscription details + const subscriptions = await WebhookService.listSubscriptions(organizationId); + const subscription = subscriptions.find(s => s.id === subscriptionId); + + if (!subscription) { + throw new Error(`Subscription ${subscriptionId} not found for organization ${organizationId}`); + } + + // Get subscription metrics + const result = await pool.query( + `SELECT * FROM webhook_delivery_health_summary + WHERE organization_id = $1 AND subscription_id = $2 + AND last_attempt >= $3`, + [organizationId, subscriptionId, range.start] + ); + + if (result.rows.length === 0) { + // Return empty summary if no metrics found + return { + subscriptionId, + url: subscription.url, + events: subscription.events, + organizationId, + totalAttempts: 0, + successfulAttempts: 0, + failedAttempts: 0, + pendingRetries: 0, + successRatePercent: 0, + avgResponseTimeMs: undefined, + firstAttempt: new Date(), + lastAttempt: new Date(), + mostCommonErrorCode: undefined, + recentFailures24h: 0, + }; + } + + const metrics = result.rows[0]; + + return { + subscriptionId, + url: subscription.url, + events: subscription.events, + organizationId, + totalAttempts: parseInt(metrics.total_attempts, 10) || 0, + successfulAttempts: parseInt(metrics.successful_attempts, 10) || 0, + failedAttempts: parseInt(metrics.failed_attempts, 10) || 0, + pendingRetries: parseInt(metrics.pending_retries, 10) || 0, + successRatePercent: parseFloat(metrics.success_rate_percent) || 0, + avgResponseTimeMs: metrics.avg_response_time_ms ? parseFloat(metrics.avg_response_time_ms) : undefined, + firstAttempt: new Date(metrics.first_attempt), + lastAttempt: new Date(metrics.last_attempt), + mostCommonErrorCode: metrics.most_common_error_code || undefined, + recentFailures24h: parseInt(metrics.recent_failures_24h, 10) || 0, + }; + } catch (error) { + logger.error('Failed to get subscription summary', { + error, + organizationId, + subscriptionId + }); + throw new Error(`Failed to get subscription summary: ${error.message}`); + } + } + + /** + * Get summary for a specific event type + */ + async getEventTypeSummary( + organizationId: number, + eventType: string, + timeRange?: { start: Date; end: Date } + ): Promise { + try { + const range = timeRange || this.getDefaultTimeRange(); + + const result = await pool.query( + `SELECT + event_type, + COUNT(*) as total_attempts, + COUNT(CASE WHEN status = 'success' THEN 1 END) as successful_attempts, + COUNT(CASE WHEN status = 'failure' THEN 1 END) as failed_attempts, + AVG(response_time_ms) as avg_response_time_ms, + COUNT(DISTINCT subscription_id) as subscriptions_count + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND event_type = $2 + AND created_at BETWEEN $3 AND $4 + GROUP BY event_type`, + [organizationId, eventType, range.start, range.end] + ); + + if (result.rows.length === 0) { + return { + eventType, + totalAttempts: 0, + successfulAttempts: 0, + failedAttempts: 0, + successRatePercent: 0, + avgResponseTimeMs: undefined, + subscriptionsCount: 0, + }; + } + + const row = result.rows[0]; + const totalAttempts = parseInt(row.total_attempts, 10) || 0; + const successfulAttempts = parseInt(row.successful_attempts, 10) || 0; + const successRatePercent = totalAttempts > 0 + ? (successfulAttempts / totalAttempts) * 100 + : 0; + + return { + eventType, + totalAttempts, + successfulAttempts, + failedAttempts: parseInt(row.failed_attempts, 10) || 0, + successRatePercent, + avgResponseTimeMs: row.avg_response_time_ms ? parseFloat(row.avg_response_time_ms) : undefined, + subscriptionsCount: parseInt(row.subscriptions_count, 10) || 0, + }; + } catch (error) { + logger.error('Failed to get event type summary', { + error, + organizationId, + eventType + }); + throw new Error(`Failed to get event type summary: ${error.message}`); + } + } + + // Private helper methods + + private getDefaultTimeRange(): { start: Date; end: Date } { + const end = new Date(); + const start = new Date(); + start.setDate(start.getDate() - 7); // Last 7 days by default + return { start, end }; + } + + private async getOverallStats( + organizationId: number, + timeRange: { start: Date; end: Date } + ) { + const result = await pool.query( + `SELECT + COUNT(*) as total_attempts, + COUNT(CASE WHEN status = 'success' THEN 1 END) as successful_attempts, + COUNT(CASE WHEN status = 'failure' THEN 1 END) as failed_attempts, + COUNT(CASE WHEN status = 'retry_scheduled' THEN 1 END) as pending_retries, + AVG(response_time_ms) as avg_response_time_ms + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND created_at BETWEEN $2 AND $3`, + [organizationId, timeRange.start, timeRange.end] + ); + + const row = result.rows[0] || {}; + const totalAttempts = parseInt(row.total_attempts, 10) || 0; + const successfulAttempts = parseInt(row.successful_attempts, 10) || 0; + const successRatePercent = totalAttempts > 0 + ? (successfulAttempts / totalAttempts) * 100 + : 0; + + return { + totalAttempts, + successfulAttempts, + failedAttempts: parseInt(row.failed_attempts, 10) || 0, + pendingRetries: parseInt(row.pending_retries, 10) || 0, + overallSuccessRatePercent: successRatePercent, + avgResponseTimeMs: row.avg_response_time_ms ? parseFloat(row.avg_response_time_ms) : undefined, + }; + } + + private async getSubscriptionBreakdown( + organizationId: number, + subscriptions: any[], + timeRange: { start: Date; end: Date }, + maxSubscriptions: number + ): Promise { + if (subscriptions.length === 0) { + return []; + } + + const subscriptionIds = subscriptions.map(s => s.id); + const placeholders = subscriptionIds.map((_, i) => `$${i + 2}`).join(','); + + const result = await pool.query( + `SELECT * FROM webhook_delivery_health_summary + WHERE organization_id = $1 + AND subscription_id IN (${placeholders}) + AND last_attempt >= $${subscriptionIds.length + 2} + ORDER BY success_rate_percent ASC, total_attempts DESC + LIMIT $${subscriptionIds.length + 3}`, + [organizationId, ...subscriptionIds, timeRange.start, maxSubscriptions] + ); + + return result.rows.map(row => { + const subscription = subscriptions.find(s => s.id === row.subscription_id); + + return { + subscriptionId: row.subscription_id, + url: subscription?.url || row.url, + events: subscription?.events || [], + organizationId, + totalAttempts: parseInt(row.total_attempts, 10) || 0, + successfulAttempts: parseInt(row.successful_attempts, 10) || 0, + failedAttempts: parseInt(row.failed_attempts, 10) || 0, + pendingRetries: parseInt(row.pending_retries, 10) || 0, + successRatePercent: parseFloat(row.success_rate_percent) || 0, + avgResponseTimeMs: row.avg_response_time_ms ? parseFloat(row.avg_response_time_ms) : undefined, + firstAttempt: new Date(row.first_attempt), + lastAttempt: new Date(row.last_attempt), + mostCommonErrorCode: row.most_common_error_code || undefined, + recentFailures24h: parseInt(row.recent_failures_24h, 10) || 0, + }; + }); + } + + private async getEventTypeBreakdown( + organizationId: number, + timeRange: { start: Date; end: Date }, + maxEventTypes: number + ): Promise { + const result = await pool.query( + `SELECT + event_type, + COUNT(*) as total_attempts, + COUNT(CASE WHEN status = 'success' THEN 1 END) as successful_attempts, + COUNT(CASE WHEN status = 'failure' THEN 1 END) as failed_attempts, + AVG(response_time_ms) as avg_response_time_ms, + COUNT(DISTINCT subscription_id) as subscriptions_count + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND created_at BETWEEN $2 AND $3 + GROUP BY event_type + ORDER BY total_attempts DESC + LIMIT $4`, + [organizationId, timeRange.start, timeRange.end, maxEventTypes] + ); + + return result.rows.map(row => { + const totalAttempts = parseInt(row.total_attempts, 10) || 0; + const successfulAttempts = parseInt(row.successful_attempts, 10) || 0; + const successRatePercent = totalAttempts > 0 + ? (successfulAttempts / totalAttempts) * 100 + : 0; + + return { + eventType: row.event_type, + totalAttempts, + successfulAttempts, + failedAttempts: parseInt(row.failed_attempts, 10) || 0, + successRatePercent, + avgResponseTimeMs: row.avg_response_time_ms ? parseFloat(row.avg_response_time_ms) : undefined, + subscriptionsCount: parseInt(row.subscriptions_count, 10) || 0, + }; + }); + } + + private async analyzeFailurePatterns( + organizationId: number, + timeRange: { start: Date; end: Date } + ) { + // Get most common error codes + const errorCodesResult = await pool.query( + `SELECT + error_code, + COUNT(*) as count + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND created_at BETWEEN $2 AND $3 + AND status = 'failure' + AND error_code IS NOT NULL + GROUP BY error_code + ORDER BY count DESC + LIMIT 10`, + [organizationId, timeRange.start, timeRange.end] + ); + + const totalFailures = errorCodesResult.rows.reduce((sum, row) => sum + parseInt(row.count, 10), 0); + const mostCommonErrorCodes = errorCodesResult.rows.map(row => ({ + errorCode: row.error_code, + count: parseInt(row.count, 10), + percentage: totalFailures > 0 ? (parseInt(row.count, 10) / totalFailures) * 100 : 0, + })); + + // Get recurring failures (same subscription, same error multiple times) + const recurringResult = await pool.query( + `SELECT + subscription_id, + url, + error_code, + COUNT(*) as failure_count, + MAX(created_at) as last_failure + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND created_at BETWEEN $2 AND $3 + AND status = 'failure' + AND error_code IS NOT NULL + GROUP BY subscription_id, url, error_code + HAVING COUNT(*) > 1 + ORDER BY failure_count DESC`, + [organizationId, timeRange.start, timeRange.end] + ); + + const recurringFailures = recurringResult.rows.map(row => ({ + subscriptionId: row.subscription_id, + url: row.url, + errorCode: row.error_code, + failureCount: parseInt(row.failure_count, 10), + lastFailure: new Date(row.last_failure), + })); + + // Categorize failures + const failureTypesResult = await pool.query( + `SELECT + COUNT(CASE WHEN error_code = 'TIMEOUT' THEN 1 END) as timeout_failures, + COUNT(CASE WHEN error_code IN ('CONNECTION_REFUSED', 'DNS_ERROR') THEN 1 END) as network_error_failures, + COUNT(CASE WHEN error_code LIKE '%ERROR' AND error_code NOT IN ('TIMEOUT', 'CONNECTION_REFUSED', 'DNS_ERROR') THEN 1 END) as http_error_failures + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND created_at BETWEEN $2 AND $3 + AND status = 'failure'`, + [organizationId, timeRange.start, timeRange.end] + ); + + const failureTypes = failureTypesResult.rows[0] || {}; + + return { + mostCommonErrorCodes, + recurringFailures, + timeoutFailures: parseInt(failureTypes.timeout_failures, 10) || 0, + networkErrorFailures: parseInt(failureTypes.network_error_failures, 10) || 0, + httpErrorFailures: parseInt(failureTypes.http_error_failures, 10) || 0, + }; + } + + private async getHourlyTrends( + organizationId: number, + timeRange: { start: Date; end: Date } + ) { + const result = await pool.query( + `SELECT + DATE_TRUNC('hour', created_at) as hour, + COUNT(*) as attempts, + COUNT(CASE WHEN status = 'success' THEN 1 END) as successful_attempts, + AVG(response_time_ms) as avg_response_time_ms + FROM webhook_delivery_metrics + WHERE organization_id = $1 + AND created_at BETWEEN $2 AND $3 + GROUP BY DATE_TRUNC('hour', created_at) + ORDER BY hour DESC`, + [organizationId, timeRange.start, timeRange.end] + ); + + return result.rows.map(row => { + const attempts = parseInt(row.attempts, 10) || 0; + const successfulAttempts = parseInt(row.successful_attempts, 10) || 0; + const successRatePercent = attempts > 0 + ? (successfulAttempts / attempts) * 100 + : 0; + + return { + hour: row.hour.toISOString(), + attempts, + successRatePercent, + avgResponseTimeMs: row.avg_response_time_ms ? parseFloat(row.avg_response_time_ms) : undefined, + }; + }); + } + + private generateRecommendations( + subscriptions: WebhookSubscriptionSummary[], + eventTypes: WebhookEventTypeSummary[], + failurePatterns: any + ) { + const recommendations: Array<{ + type: 'warning' | 'suggestion' | 'critical'; + message: string; + action?: string; + }> = []; + + // Check for critical failures (0% success rate) + const criticalSubscriptions = subscriptions.filter( + sub => sub.totalAttempts > 0 && sub.successRatePercent === 0 + ); + + criticalSubscriptions.forEach(sub => { + recommendations.push({ + type: 'critical', + message: `Subscription ${sub.subscriptionId} has 0% success rate with ${sub.recentFailures24h} recent failures.`, + action: `Review webhook endpoint configuration for ${sub.url}`, + }); + }); + + // Check for warning-level failures (< 80% success rate) + const warningSubscriptions = subscriptions.filter( + sub => sub.totalAttempts > 10 && sub.successRatePercent < 80 && sub.successRatePercent > 0 + ); + + warningSubscriptions.forEach(sub => { + recommendations.push({ + type: 'warning', + message: `Subscription ${sub.subscriptionId} has low success rate (${sub.successRatePercent.toFixed(1)}%).`, + action: 'Monitor for improvement or investigate endpoint reliability', + }); + }); + + // Check for recurring error patterns + if (failurePatterns.recurringFailures.length > 0) { + failurePatterns.recurringFailures.forEach((failure: any) => { + if (failure.failureCount >= 3) { + recommendations.push({ + type: 'critical', + message: `Recurring ${failure.errorCode} errors (${failure.failureCount} failures) for subscription ${failure.subscriptionId}.`, + action: `Investigate endpoint ${failure.url} for persistent issues`, + }); + } + }); + } + + // Check for timeout failures + if (failurePatterns.timeoutFailures > 5) { + recommendations.push({ + type: 'warning', + message: `Multiple timeout failures detected (${failurePatterns.timeoutFailures}). Consider increasing timeout settings.`, + action: 'Review webhook timeout configuration', + }); + } + + // Check for slow response times + const slowSubscriptions = subscriptions.filter( + sub => sub.avgResponseTimeMs && sub.avgResponseTimeMs > 1000 + ); + + if (slowSubscriptions.length > 0) { + recommendations.push({ + type: 'suggestion', + message: `${slowSubscriptions.length} subscription(s) have average response times over 1 second.`, + action: 'Consider optimizing webhook receiver performance', + }); + } + + // Check for retry patterns + const highRetrySubscriptions = subscriptions.filter( + sub => sub.pendingRetries > 0 + ); + + if (highRetrySubscriptions.length > 0) { + recommendations.push({ + type: 'suggestion', + message: `${highRetrySubscriptions.length} subscription(s) have pending retries.`, + action: 'Review retry strategy and exponential backoff configuration', + }); + } + + // If no critical issues, add general suggestions + if (recommendations.length === 0) { + recommendations.push({ + type: 'suggestion', + message: 'Webhook delivery health is good. Consider setting up proactive monitoring alerts.', + action: 'Configure alerting for success rate drops below 95%', + }); + } + + return recommendations; + } +} + +export const webhookHealthReportService = new WebhookHealthReportService(); \ No newline at end of file