feat(webhook): add optional X-Ray instrumentation for GitHub event delivery latency

lambdas/functions/webhook/src/lambda.ts

@@ -9,15 +9,22 @@ import { EventWrapper } from './types';
 import { WorkflowJobEvent } from '@octokit/webhooks-types';
 import { ConfigDispatcher, ConfigWebhook, ConfigWebhookEventBridge } from './ConfigLoader';
 import { dispatch } from './runners/dispatch';
+import { githubEventTracingMiddleware } from './tracing/githubEventTracingMiddleware';
 
 export interface Response {
   statusCode: number;
   body: string;
 }
 
-middy(directWebhook).use(captureLambdaHandler(tracer));
+export const directWebhook = middy(directWebhookHandler)
+  .use(captureLambdaHandler(tracer))
+  .use(githubEventTracingMiddleware());
 
-export async function directWebhook(event: APIGatewayEvent, context: Context): Promise<Response> {
+export const eventBridgeWebhook = middy(eventBridgeWebhookHandler)
+  .use(captureLambdaHandler(tracer))
+  .use(githubEventTracingMiddleware());
+
+async function directWebhookHandler(event: APIGatewayEvent, context: Context): Promise<Response> {
   setContext(context, 'lambda.ts');
   logger.logEventIfEnabled(event);
 
@@ -42,7 +49,7 @@ export async function directWebhook(event: APIGatewayEvent, context: Context): P
   return result;
 }
 
-export async function eventBridgeWebhook(event: APIGatewayEvent, context: Context): Promise<Response> {
+async function eventBridgeWebhookHandler(event: APIGatewayEvent, context: Context): Promise<Response> {
   setContext(context, 'lambda.ts');
   logger.logEventIfEnabled(event);
 

lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.test.ts

@@ -0,0 +1,86 @@
+import { tracer } from '@aws-github-runner/aws-powertools-util';
+import { APIGatewayEvent, Context } from 'aws-lambda';
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+import { githubEventTracingMiddleware } from './githubEventTracingMiddleware';
+
+describe('githubEventTracingMiddleware', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  function buildEvent(overrides: Partial<APIGatewayEvent> = {}): APIGatewayEvent {
+    return {
+      headers: {
+        'X-GitHub-Event': 'workflow_job',
+        'X-GitHub-Delivery': 'delivery-id-1',
+      },
+      body: JSON.stringify({ workflow_job: { created_at: new Date(Date.now() - 5000).toISOString() } }),
+      requestContext: { requestTimeEpoch: Date.now() - 20 },
+      ...overrides,
+    } as unknown as APIGatewayEvent;
+  }
+
+  it('does nothing when tracing is not enabled (no active segment)', async () => {
+    vi.spyOn(tracer, 'getSegment').mockReturnValue(undefined);
+    const putAnnotation = vi.spyOn(tracer, 'putAnnotation');
+
+    const { before, after } = githubEventTracingMiddleware();
+    await before?.({ event: buildEvent(), context: {} as Context } as never);
+    await after?.({ event: buildEvent(), context: {} as Context } as never);
+
+    expect(putAnnotation).not.toHaveBeenCalled();
+  });
+
+  it('annotates event type, delivery id, ingress lag and workflow_job age when tracing is active', async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any);
+    const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined);
+
+    const { before } = githubEventTracingMiddleware();
+    await before?.({ event: buildEvent(), context: {} as Context } as never);
+
+    expect(putAnnotation).toHaveBeenCalledWith('github_event_type', 'workflow_job');
+    expect(putAnnotation).toHaveBeenCalledWith('github_delivery_id', 'delivery-id-1');
+    expect(putAnnotation).toHaveBeenCalledWith('api_gateway_ingress_to_lambda_ms', expect.any(Number));
+    expect(putAnnotation).toHaveBeenCalledWith('workflow_job_age_ms', expect.any(Number));
+  });
+
+  it('skips workflow_job_age_ms for non workflow_job event types', async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any);
+    const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined);
+
+    const { before } = githubEventTracingMiddleware();
+    await before?.({
+      event: buildEvent({ headers: { 'X-GitHub-Event': 'push', 'X-GitHub-Delivery': 'delivery-id-2' } }),
+      context: {} as Context,
+    } as never);
+
+    expect(putAnnotation).toHaveBeenCalledWith('github_event_type', 'push');
+    expect(putAnnotation).not.toHaveBeenCalledWith('workflow_job_age_ms', expect.any(Number));
+  });
+
+  it('does not throw on a malformed body and skips the age annotation', async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any);
+    const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined);
+
+    const { before } = githubEventTracingMiddleware();
+    await before?.({ event: buildEvent({ body: 'not-json' }), context: {} as Context } as never);
+
+    expect(putAnnotation).not.toHaveBeenCalledWith('workflow_job_age_ms', expect.any(Number));
+  });
+
+  it('adds lambda_processing_ms annotation on after and onError', async () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any);
+    const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined);
+
+    const middleware = githubEventTracingMiddleware();
+    await middleware.before?.({ event: buildEvent(), context: {} as Context } as never);
+    await middleware.after?.({ event: buildEvent(), context: {} as Context } as never);
+
+    expect(putAnnotation).toHaveBeenCalledWith('lambda_processing_ms', expect.any(Number));
+  });
+});

lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.ts

@@ -0,0 +1,62 @@
+import { MiddlewareObj } from '@middy/core';
+import { APIGatewayEvent, Context } from 'aws-lambda';
+import { tracer } from '@aws-github-runner/aws-powertools-util';
+
+// Reusable X-Ray instrumentation for the webhook Lambda's GitHub-originated event types.
+// Adds annotations (queryable/alertable) rather than synthetic subsegments, so it never
+// alters the shape of the actual X-Ray trace for this invocation.
+export function githubEventTracingMiddleware<TResult = unknown>(): MiddlewareObj<APIGatewayEvent, TResult> {
+  let lambdaStartedAt: number;
+
+  const before = (request: { event: APIGatewayEvent; context: Context }): void => {
+    lambdaStartedAt = Date.now();
+
+    if (!tracer.getSegment()) return;
+
+    const headers = lowerCaseKeys(request.event.headers as Record<string, string | undefined>);
+    const eventType = headers['x-github-event'];
+    const deliveryId = headers['x-github-delivery'];
+
+    if (eventType !== undefined) tracer.putAnnotation('github_event_type', eventType);
+    if (deliveryId !== undefined) tracer.putAnnotation('github_delivery_id', deliveryId);
+
+    const apiGatewayIngressAtMs = request.event.requestContext?.requestTimeEpoch;
+    if (apiGatewayIngressAtMs !== undefined) {
+      tracer.putAnnotation('api_gateway_ingress_to_lambda_ms', lambdaStartedAt - apiGatewayIngressAtMs);
+    }
+
+    if (eventType === 'workflow_job') {
+      const workflowJobAgeMs = tryGetWorkflowJobAgeMs(request.event.body, lambdaStartedAt);
+      if (workflowJobAgeMs !== undefined) {
+        tracer.putAnnotation('workflow_job_age_ms', workflowJobAgeMs);
+      }
+    }
+  };
+
+  const after = (): void => {
+    if (!tracer.getSegment()) return;
+    tracer.putAnnotation('lambda_processing_ms', Date.now() - lambdaStartedAt);
+  };
+
+  return { before, after, onError: after };
+}
+
+function lowerCaseKeys(headers: Record<string, string | undefined>): Record<string, string | undefined> {
+  const result: Record<string, string | undefined> = {};
+  for (const key in headers) {
+    result[key.toLowerCase()] = headers[key];
+  }
+  return result;
+}
+
+// Best-effort: the body isn't verified/parsed yet at this point in the request lifecycle,
+// so a malformed or not-yet-signature-verified payload must not fail the request.
+function tryGetWorkflowJobAgeMs(body: string | null, nowMs: number): number | undefined {
+  if (!body) return undefined;
+  try {
+    const createdAt = (JSON.parse(body) as { workflow_job?: { created_at?: string } }).workflow_job?.created_at;
+    return createdAt ? nowMs - new Date(createdAt).getTime() : undefined;
+  } catch {
+    return undefined;
+  }
+}