diff --git a/.claude/scripts/test-api-logs.ts b/.claude/scripts/test-api-logs.ts new file mode 100644 index 0000000..f4419a3 --- /dev/null +++ b/.claude/scripts/test-api-logs.ts @@ -0,0 +1,124 @@ +#!/usr/bin/env bun +import { desc, eq } from 'drizzle-orm' +import { db } from '#/db' +import { apiCallLogs } from '#/db/schema' +import { + loggedFetch, + withApiLogContext, +} from '#/services/logs/logged-fetch' + +const SENTINEL_HOST = 'https://example.test/api-logs-smoke' + +async function clearOldRuns(): Promise { + // Best-effort cleanup so repeat runs don't pollute the table. + await db + .delete(apiCallLogs) + .where(eq(apiCallLogs.provider, 'openalex')) + .returning({ id: apiCallLogs.id }) + .catch(() => []) +} + +function stubFetch(body: string, status: number): void { + globalThis.fetch = async () => + new Response(body, { + status, + headers: { + 'content-type': 'application/json', + 'content-length': String(body.length), + }, + }) +} + +function restoreFetch(original: typeof globalThis.fetch): void { + globalThis.fetch = original +} + +const originalFetch = globalThis.fetch + +async function main(): Promise { + console.log('Smoke-test for loggedFetch + api_call_logs') + console.log('---') + + await clearOldRuns() + + // Case 1: success with body capture + stubFetch('{"results":[{"id":"W123"}]}', 200) + const fakeJobId = '00000000-0000-0000-0000-000000000000' + try { + await withApiLogContext({ trackJobId: fakeJobId }, async () => { + const res = await loggedFetch( + { provider: 'openalex' }, + `${SENTINEL_HOST}/works?search=test`, + ) + const body = await res.json() + console.log('case 1: caller got body', JSON.stringify(body)) + }) + } catch (err) { + // trackJobId references a fake uuid that doesn't exist in jobs table — + // the FK will reject the insert. That's expected; we'll verify by + // dropping the FK constraint, or by using a real job. For the smoke + // test we just verify the wrapper doesn't break the caller. + console.log('case 1 caller still completed despite FK error:', err) + } + + // Case 2: HTTP error + stubFetch('{"error":"forbidden"}', 403) + const res2 = await loggedFetch( + { provider: 'openalex' }, + `${SENTINEL_HOST}/works/forbidden`, + ) + console.log('case 2: status', res2.status) + + // Case 3: network error + globalThis.fetch = async () => { + throw new Error('econnrefused') + } + try { + await loggedFetch( + { provider: 'openalex' }, + `${SENTINEL_HOST}/works/network-fail`, + ) + console.log('case 3 FAILED — should have thrown') + } catch (err) { + console.log('case 3: caller saw network error:', (err as Error).message) + } + + restoreFetch(originalFetch) + + // Wait for fire-and-forget DB writes. + await new Promise((resolve) => setTimeout(resolve, 1000)) + + // Verify rows landed for the calls that didn't FK-fail. + const rows = await db + .select() + .from(apiCallLogs) + .where(eq(apiCallLogs.provider, 'openalex')) + .orderBy(desc(apiCallLogs.createdAt)) + .limit(10) + + console.log('---') + console.log(`Found ${rows.length} log rows (provider='openalex'):`) + for (const row of rows) { + console.log( + ` [${row.id}] ${row.outcome} status=${row.status} duration=${row.durationMs}ms url=${row.url}`, + ) + if (row.bodyPreview) { + const preview = row.bodyPreview.slice(0, 80) + console.log( + ` body (${row.bodySizeBytes}B${row.bodyTruncated ? ', truncated' : ''}): ${preview}`, + ) + } + if (row.errorMessage) { + console.log(` error: ${row.errorMessage}`) + } + } + + console.log('---') + console.log( + rows.length >= 2 + ? 'PASS: at least the http_error + network_error cases landed in the DB.' + : 'WARN: fewer rows than expected — check api_call_logs migration was applied (bun run db:migrate or restart docker compose).', + ) +} + +await main() diff --git a/drizzle/0008_solid_venus.sql b/drizzle/0008_solid_venus.sql new file mode 100644 index 0000000..b2eb062 --- /dev/null +++ b/drizzle/0008_solid_venus.sql @@ -0,0 +1,25 @@ +CREATE TYPE "public"."api_call_outcome" AS ENUM('success', 'http_error', 'network_error', 'timeout');--> statement-breakpoint +CREATE TABLE "api_call_logs" ( + "id" serial PRIMARY KEY NOT NULL, + "track_job_id" uuid, + "eval_job_id" uuid, + "provider" text NOT NULL, + "method" text DEFAULT 'GET' NOT NULL, + "url" text NOT NULL, + "status" integer, + "response_headers" jsonb, + "body_preview" text, + "body_truncated" boolean DEFAULT false NOT NULL, + "body_size_bytes" integer, + "outcome" "api_call_outcome" NOT NULL, + "error_message" text, + "duration_ms" integer NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "api_call_logs" ADD CONSTRAINT "api_call_logs_track_job_id_jobs_id_fk" FOREIGN KEY ("track_job_id") REFERENCES "public"."jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "api_call_logs" ADD CONSTRAINT "api_call_logs_eval_job_id_evaluation_jobs_id_fk" FOREIGN KEY ("eval_job_id") REFERENCES "public"."evaluation_jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "api_call_logs_created_idx" ON "api_call_logs" USING btree ("created_at");--> statement-breakpoint +CREATE INDEX "api_call_logs_provider_created_idx" ON "api_call_logs" USING btree ("provider","created_at");--> statement-breakpoint +CREATE INDEX "api_call_logs_track_job_idx" ON "api_call_logs" USING btree ("track_job_id");--> statement-breakpoint +CREATE INDEX "api_call_logs_eval_job_idx" ON "api_call_logs" USING btree ("eval_job_id"); \ No newline at end of file diff --git a/drizzle/meta/0008_snapshot.json b/drizzle/meta/0008_snapshot.json new file mode 100644 index 0000000..75b4680 --- /dev/null +++ b/drizzle/meta/0008_snapshot.json @@ -0,0 +1,2124 @@ +{ + "id": "e431a731-3cbf-49d5-87a1-b3d433e3e444", + "prevId": "5dbe942a-cf99-43e9-8fae-499d42a89526", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_call_logs": { + "name": "api_call_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "track_job_id": { + "name": "track_job_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "eval_job_id": { + "name": "eval_job_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'GET'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "response_headers": { + "name": "response_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "body_preview": { + "name": "body_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body_truncated": { + "name": "body_truncated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "body_size_bytes": { + "name": "body_size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "api_call_outcome", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "api_call_logs_created_idx": { + "name": "api_call_logs_created_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_call_logs_provider_created_idx": { + "name": "api_call_logs_provider_created_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_call_logs_track_job_idx": { + "name": "api_call_logs_track_job_idx", + "columns": [ + { + "expression": "track_job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_call_logs_eval_job_idx": { + "name": "api_call_logs_eval_job_idx", + "columns": [ + { + "expression": "eval_job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_call_logs_track_job_id_jobs_id_fk": { + "name": "api_call_logs_track_job_id_jobs_id_fk", + "tableFrom": "api_call_logs", + "tableTo": "jobs", + "columnsFrom": [ + "track_job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_call_logs_eval_job_id_evaluation_jobs_id_fk": { + "name": "api_call_logs_eval_job_id_evaluation_jobs_id_fk", + "tableFrom": "api_call_logs", + "tableTo": "evaluation_jobs", + "columnsFrom": [ + "eval_job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.citation_matches": { + "name": "citation_matches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "citation_key": { + "name": "citation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "match_type": { + "name": "match_type", + "type": "match_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'unmatched'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "citation_matches_job_key_idx": { + "name": "citation_matches_job_key_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "citation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "citation_matches_job_id_jobs_id_fk": { + "name": "citation_matches_job_id_jobs_id_fk", + "tableFrom": "citation_matches", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "citation_matches_reference_id_references_id_fk": { + "name": "citation_matches_reference_id_references_id_fk", + "tableFrom": "citation_matches", + "tableTo": "references", + "columnsFrom": [ + "reference_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.citations": { + "name": "citations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "citation_key": { + "name": "citation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thesis_page": { + "name": "thesis_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "thesis_context": { + "name": "thesis_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "raw_match": { + "name": "raw_match", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "citations_job_idx": { + "name": "citations_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "citations_job_id_jobs_id_fk": { + "name": "citations_job_id_jobs_id_fk", + "tableFrom": "citations", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.configurations": { + "name": "configurations", + "schema": "", + "columns": { + "code": { + "name": "code", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dictionary": { + "name": "dictionary", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "word": { + "name": "word", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "arti": { + "name": "arti", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "dictionary_word_lookup_idx": { + "name": "dictionary_word_lookup_idx", + "columns": [ + { + "expression": "lower(trim(\"word\"))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dictionary_cache": { + "name": "dictionary_cache", + "schema": "", + "columns": { + "word": { + "name": "word", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "found": { + "name": "found", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "arti": { + "name": "arti", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.evaluation_findings": { + "name": "evaluation_findings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "eval_job_id": { + "name": "eval_job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "evaluation_category", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "evaluation_severity", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'warning'" + }, + "page_number": { + "name": "page_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "offset": { + "name": "offset", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "length": { + "name": "length", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "excerpt": { + "name": "excerpt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggestion": { + "name": "suggestion", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_findings_job_page_idx": { + "name": "evaluation_findings_job_page_idx", + "columns": [ + { + "expression": "eval_job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "page_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluation_findings_job_cat_sev_idx": { + "name": "evaluation_findings_job_cat_sev_idx", + "columns": [ + { + "expression": "eval_job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "severity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluation_findings_eval_job_id_evaluation_jobs_id_fk": { + "name": "evaluation_findings_eval_job_id_evaluation_jobs_id_fk", + "tableFrom": "evaluation_findings", + "tableTo": "evaluation_jobs", + "columnsFrom": [ + "eval_job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.evaluation_jobs": { + "name": "evaluation_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "status": { + "name": "status", + "type": "evaluation_job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "total_pages": { + "name": "total_pages", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "extracted_pages": { + "name": "extracted_pages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "current_step": { + "name": "current_step", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kbbi_progress": { + "name": "kbbi_progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "kbbi_total": { + "name": "kbbi_total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "eyd_progress": { + "name": "eyd_progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "eyd_total": { + "name": "eyd_total", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.evaluation_pages": { + "name": "evaluation_pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "eval_job_id": { + "name": "eval_job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "page_number": { + "name": "page_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "low_text_density": { + "name": "low_text_density", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "code_ranges": { + "name": "code_ranges", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "italic_ranges": { + "name": "italic_ranges", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_pages_job_page_idx": { + "name": "evaluation_pages_job_page_idx", + "columns": [ + { + "expression": "eval_job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "page_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluation_pages_eval_job_id_evaluation_jobs_id_fk": { + "name": "evaluation_pages_eval_job_id_evaluation_jobs_id_fk", + "tableFrom": "evaluation_pages", + "tableTo": "evaluation_jobs", + "columnsFrom": [ + "eval_job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.evaluation_summary": { + "name": "evaluation_summary", + "schema": "", + "columns": { + "eval_job_id": { + "name": "eval_job_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "kbbi_error_count": { + "name": "kbbi_error_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "eyd_error_count": { + "name": "eyd_error_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "overall_score": { + "name": "overall_score", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "raw_report": { + "name": "raw_report", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "evaluation_summary_eval_job_id_evaluation_jobs_id_fk": { + "name": "evaluation_summary_eval_job_id_evaluation_jobs_id_fk", + "tableFrom": "evaluation_summary", + "tableTo": "evaluation_jobs", + "columnsFrom": [ + "eval_job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.evaluation_vocabulary": { + "name": "evaluation_vocabulary", + "schema": "", + "columns": { + "word": { + "name": "word", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "classification": { + "name": "classification", + "type": "vocabulary_classification", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "status": { + "name": "status", + "type": "job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "total_pages": { + "name": "total_pages", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "extracted_pages": { + "name": "extracted_pages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pages": { + "name": "pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "page_number": { + "name": "page_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "low_text_density": { + "name": "low_text_density", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "pages_job_page_idx": { + "name": "pages_job_page_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "page_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pages_job_id_jobs_id_fk": { + "name": "pages_job_id_jobs_id_fk", + "tableFrom": "pages", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passage_match_batches": { + "name": "passage_match_batches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "batch_index": { + "name": "batch_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_pdf_id": { + "name": "source_pdf_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "passage_batch_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "citation_count": { + "name": "citation_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "matched_count": { + "name": "matched_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "no_match_count": { + "name": "no_match_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "passage_match_batches_job_batch_idx": { + "name": "passage_match_batches_job_batch_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "batch_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passage_match_batches_job_status_idx": { + "name": "passage_match_batches_job_status_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passage_match_batches_job_id_jobs_id_fk": { + "name": "passage_match_batches_job_id_jobs_id_fk", + "tableFrom": "passage_match_batches", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "passage_match_batches_source_pdf_id_source_pdfs_id_fk": { + "name": "passage_match_batches_source_pdf_id_source_pdfs_id_fk", + "tableFrom": "passage_match_batches", + "tableTo": "source_pdfs", + "columnsFrom": [ + "source_pdf_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passage_matches": { + "name": "passage_matches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "citation_id": { + "name": "citation_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_pdf_id": { + "name": "source_pdf_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_page": { + "name": "source_page", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "matched_passage": { + "name": "matched_passage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reasoning": { + "name": "reasoning", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "passage_matches_job_idx": { + "name": "passage_matches_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passage_matches_citation_idx": { + "name": "passage_matches_citation_idx", + "columns": [ + { + "expression": "citation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passage_matches_job_id_jobs_id_fk": { + "name": "passage_matches_job_id_jobs_id_fk", + "tableFrom": "passage_matches", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "passage_matches_citation_id_citations_id_fk": { + "name": "passage_matches_citation_id_citations_id_fk", + "tableFrom": "passage_matches", + "tableTo": "citations", + "columnsFrom": [ + "citation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "passage_matches_source_pdf_id_source_pdfs_id_fk": { + "name": "passage_matches_source_pdf_id_source_pdfs_id_fk", + "tableFrom": "passage_matches", + "tableTo": "source_pdfs", + "columnsFrom": [ + "source_pdf_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.references": { + "name": "references", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "year": { + "name": "year", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "doi": { + "name": "doi", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "publisher": { + "name": "publisher", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "journal": { + "name": "journal", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "raw_text": { + "name": "raw_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "start_page": { + "name": "start_page", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "references_job_idx": { + "name": "references_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "references_job_id_jobs_id_fk": { + "name": "references_job_id_jobs_id_fk", + "tableFrom": "references", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_pages": { + "name": "source_pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "source_pdf_id": { + "name": "source_pdf_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "page_number": { + "name": "page_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "char_count": { + "name": "char_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_pages_pdf_page_idx": { + "name": "source_pages_pdf_page_idx", + "columns": [ + { + "expression": "source_pdf_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "page_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_pages_source_pdf_id_source_pdfs_id_fk": { + "name": "source_pages_source_pdf_id_source_pdfs_id_fk", + "tableFrom": "source_pages", + "tableTo": "source_pdfs", + "columnsFrom": [ + "source_pdf_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_pdfs": { + "name": "source_pdfs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pdf_url": { + "name": "pdf_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetch_source": { + "name": "fetch_source", + "type": "fetch_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "source_fetch_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "total_pages": { + "name": "total_pages", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_pdfs_job_idx": { + "name": "source_pdfs_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "source_pdfs_reference_idx": { + "name": "source_pdfs_reference_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_pdfs_job_id_jobs_id_fk": { + "name": "source_pdfs_job_id_jobs_id_fk", + "tableFrom": "source_pdfs", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "source_pdfs_reference_id_references_id_fk": { + "name": "source_pdfs_reference_id_references_id_fk", + "tableFrom": "source_pdfs", + "tableTo": "references", + "columnsFrom": [ + "reference_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_window_embeddings": { + "name": "source_window_embeddings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "source_pdf_id": { + "name": "source_pdf_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "page_number": { + "name": "page_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "window_idx": { + "name": "window_idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "window_text": { + "name": "window_text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "embedding": { + "name": "embedding", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "embedding_model": { + "name": "embedding_model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "embedding_dim": { + "name": "embedding_dim", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_window_embed_pdf_model_idx": { + "name": "source_window_embed_pdf_model_idx", + "columns": [ + { + "expression": "source_pdf_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "source_window_embed_unique_idx": { + "name": "source_window_embed_unique_idx", + "columns": [ + { + "expression": "source_pdf_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "embedding_model", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "page_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "window_idx", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_window_embeddings_source_pdf_id_source_pdfs_id_fk": { + "name": "source_window_embeddings_source_pdf_id_source_pdfs_id_fk", + "tableFrom": "source_window_embeddings", + "tableTo": "source_pdfs", + "columnsFrom": [ + "source_pdf_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.api_call_outcome": { + "name": "api_call_outcome", + "schema": "public", + "values": [ + "success", + "http_error", + "network_error", + "timeout" + ] + }, + "public.evaluation_category": { + "name": "evaluation_category", + "schema": "public", + "values": [ + "kbbi", + "eyd" + ] + }, + "public.evaluation_job_status": { + "name": "evaluation_job_status", + "schema": "public", + "values": [ + "pending", + "extracting", + "analyzing", + "done", + "failed" + ] + }, + "public.evaluation_severity": { + "name": "evaluation_severity", + "schema": "public", + "values": [ + "error", + "warning", + "info" + ] + }, + "public.fetch_source": { + "name": "fetch_source", + "schema": "public", + "values": [ + "doi", + "unpaywall", + "semantic-scholar", + "crossref", + "openalex", + "core", + "manual", + "europepmc", + "pubmed", + "arxiv" + ] + }, + "public.job_status": { + "name": "job_status", + "schema": "public", + "values": [ + "pending", + "extracting", + "done", + "failed" + ] + }, + "public.match_type": { + "name": "match_type", + "schema": "public", + "values": [ + "exact", + "fuzzy", + "unmatched" + ] + }, + "public.passage_batch_status": { + "name": "passage_batch_status", + "schema": "public", + "values": [ + "pending", + "running", + "done", + "failed" + ] + }, + "public.source_fetch_status": { + "name": "source_fetch_status", + "schema": "public", + "values": [ + "pending", + "found", + "downloading", + "extracting", + "done", + "failed" + ] + }, + "public.vocabulary_classification": { + "name": "vocabulary_classification", + "schema": "public", + "values": [ + "indonesian", + "english", + "tech", + "brand", + "ignore", + "typo" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 8896ef3..49586a0 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1779632857217, "tag": "0007_far_krista_starr", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1779637675421, + "tag": "0008_solid_venus", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts index 8a1dd65..af285eb 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -430,3 +430,43 @@ export const passageMatchBatches = pgTable( index('passage_match_batches_job_status_idx').on(t.jobId, t.status), ], ) + +export const apiCallOutcomeEnum = pgEnum('api_call_outcome', [ + 'success', + 'http_error', + 'network_error', + 'timeout', +]) + +export const apiCallLogs = pgTable( + 'api_call_logs', + { + id: serial().primaryKey(), + trackJobId: uuid('track_job_id').references(() => jobs.id, { + onDelete: 'cascade', + }), + evalJobId: uuid('eval_job_id').references(() => evaluationJobs.id, { + onDelete: 'cascade', + }), + provider: text().notNull(), + method: text().notNull().default('GET'), + url: text().notNull(), + status: integer(), + responseHeaders: jsonb('response_headers').$type< + Record + >(), + bodyPreview: text('body_preview'), + bodyTruncated: boolean('body_truncated').default(false).notNull(), + bodySizeBytes: integer('body_size_bytes'), + outcome: apiCallOutcomeEnum().notNull(), + errorMessage: text('error_message'), + durationMs: integer('duration_ms').notNull(), + createdAt: timestamp('created_at').defaultNow().notNull(), + }, + (t) => [ + index('api_call_logs_created_idx').on(t.createdAt), + index('api_call_logs_provider_created_idx').on(t.provider, t.createdAt), + index('api_call_logs_track_job_idx').on(t.trackJobId), + index('api_call_logs_eval_job_idx').on(t.evalJobId), + ], +) diff --git a/src/routes/admin/api-logs.tsx b/src/routes/admin/api-logs.tsx new file mode 100644 index 0000000..ca8fb61 --- /dev/null +++ b/src/routes/admin/api-logs.tsx @@ -0,0 +1,471 @@ +import { useMemo, useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { useInfiniteQuery, useQuery } from '@tanstack/react-query' +import { AlertTriangle, ChevronDown, ChevronRight } from 'lucide-react' +import { AccentInk } from '#/components/AccentWord' +import { Section } from '#/components/Section' +import { Button } from '#/components/ui/button' +import { Input } from '#/components/ui/input' +import { cn } from '#/lib/utils' +import { + getApiCallLog, + listApiCallLogs, +} from '#/services/logs/api-logs' +import { + API_PROVIDERS, + type ApiProvider, +} from '#/services/logs/logged-fetch' + +export const Route = createFileRoute('/admin/api-logs')({ + component: ApiLogsPage, + head: () => ({ meta: [{ title: 'API logs · CiteTrack' }] }), +}) + +type OutcomeFilter = 'all' | 'errors' | 'success' + +interface Filters { + providers: Set + outcome: OutcomeFilter + trackJobId: string + evalJobId: string +} + +const PAGE_SIZE = 50 + +function ApiLogsPage() { + const [filters, setFilters] = useState({ + providers: new Set(), + outcome: 'all', + trackJobId: '', + evalJobId: '', + }) + + const queryArgs = useMemo( + () => ({ + provider: + filters.providers.size > 0 ? [...filters.providers] : undefined, + outcome: filters.outcome, + trackJobId: + filters.trackJobId.trim().length > 0 + ? filters.trackJobId.trim() + : undefined, + evalJobId: + filters.evalJobId.trim().length > 0 + ? filters.evalJobId.trim() + : undefined, + limit: PAGE_SIZE, + }), + [filters], + ) + + const query = useInfiniteQuery({ + queryKey: ['api-logs', queryArgs], + initialPageParam: undefined as + | { createdAt: string; id: number } + | undefined, + queryFn: ({ pageParam }) => + listApiCallLogs({ data: { ...queryArgs, cursor: pageParam } }), + getNextPageParam: (last) => last.nextCursor ?? undefined, + }) + + const rows = useMemo( + () => query.data?.pages.flatMap((p) => p.rows) ?? [], + [query.data], + ) + + return ( +
+
+ + Admin · Diagnostik + +

+ Log API pihak ketiga +

+

+ Setiap permintaan ke OpenAlex, Crossref, KBBI, dan sumber lain + tercatat di sini lengkap dengan status, durasi, dan cuplikan body + tanggapan. Klik baris untuk melihat detail. +

+
+ +
+
+ + +
+ + + + + + + + + + + + + {rows.map((row) => ( + + ))} + {rows.length === 0 && !query.isPending && ( + + + + )} + {query.isPending && ( + + + + )} + {query.isError && ( + + + + )} + +
+ waktuproviderstatusdurasiurloutcome
+ Belum ada log. Jalankan Track atau Evaluasi dulu. +
+ Memuat. + . + . +
+ +
+
+ + {query.hasNextPage && ( +
+ +
+ )} +
+
+
+ ) +} + +function FilterBar({ + filters, + onChange, +}: { + filters: Filters + onChange: (next: Filters) => void +}) { + const toggleProvider = (p: ApiProvider) => { + const next = new Set(filters.providers) + if (next.has(p)) next.delete(p) + else next.add(p) + onChange({ ...filters, providers: next }) + } + + return ( +
+
+ provider + {API_PROVIDERS.map((p) => { + const active = filters.providers.has(p) + return ( + + ) + })} + {filters.providers.size > 0 && ( + + )} +
+ +
+ outcome + {(['all', 'errors', 'success'] as const).map((opt) => ( + + ))} +
+ +
+ + onChange({ ...filters, trackJobId: e.target.value }) + } + className="h-10 rounded-xl border-[var(--line)] bg-white font-mono text-[0.8125rem] shadow-none" + /> + + onChange({ ...filters, evalJobId: e.target.value }) + } + className="h-10 rounded-xl border-[var(--line)] bg-white font-mono text-[0.8125rem] shadow-none" + /> +
+
+ ) +} + +interface LogRowData { + id: number + createdAt: Date + provider: string + method: string + url: string + status: number | null + outcome: 'success' | 'http_error' | 'network_error' | 'timeout' + durationMs: number + errorMessage: string | null + bodySizeBytes: number | null + bodyTruncated: boolean + trackJobId: string | null + evalJobId: string | null +} + +function LogRow({ row }: { row: LogRowData }) { + const [expanded, setExpanded] = useState(false) + + return ( + <> + setExpanded((v) => !v)} + className="cursor-pointer border-t border-[var(--line)]/60 transition hover:bg-[var(--bg-cream)]/30" + > + + {expanded ? ( + + ) : ( + + )} + + + {formatTime(row.createdAt)} + + + + {row.provider} + + + + {row.status ?? '—'} + + + {row.durationMs}ms + + + {row.url} + + + + + + {expanded && } + + ) +} + +function DetailRow({ row }: { row: LogRowData }) { + const detail = useQuery({ + queryKey: ['api-log-detail', row.id], + queryFn: () => getApiCallLog({ data: { id: row.id } }), + }) + + return ( + + + + {detail.isPending && ( +

memuat detail…

+ )} + {detail.data && ( +
+ + {detail.data.errorMessage && ( +
+ {detail.data.errorMessage} +
+ )} + {detail.data.responseHeaders && ( + + )} + +
+ )} + + + ) +} + +function UrlLine({ row }: { row: { method: string; url: string } }) { + return ( +
+ + {row.method} + + {row.url} +
+ ) +} + +function HeadersBlock({ headers }: { headers: Record }) { + const entries = Object.entries(headers) + if (entries.length === 0) return null + return ( +
+

headers

+
+ {entries.map(([k, v]) => ( + + ))} +
+
+ ) +} + +function FragmentRow({ label, value }: { label: string; value: string }) { + return ( + <> +
{label}
+
{value}
+ + ) +} + +function BodyBlock({ + body, + truncated, + size, + contentType, +}: { + body: string | null + truncated: boolean + size: number | null + contentType: string | undefined +}) { + if (!body) { + return ( +

+ Body tidak direkam (binary download atau pengaturan metadata-only). +

+ ) + } + const isJson = contentType?.includes('application/json') ?? false + const rendered = isJson ? tryPrettyJson(body) : body + + return ( +
+

+ body + {size !== null && ( + + {formatBytes(size)} + {truncated && ' · dipotong'} + + )} +

+ 
+        {rendered}
+
+
+ ) +} + +function OutcomeBadge({ + outcome, +}: { + outcome: 'success' | 'http_error' | 'network_error' | 'timeout' +}) { + const tone = + outcome === 'success' ? 'info' : outcome === 'timeout' ? 'warning' : 'error' + return ( + + {outcome} + + ) +} + +function ErrorBlock({ error }: { error: unknown }) { + return ( +
+ +

+ {error instanceof Error ? error.message : 'Gagal memuat log.'} +

+
+ ) +} + +function formatTime(date: Date | string): string { + const d = typeof date === 'string' ? new Date(date) : date + const hh = String(d.getHours()).padStart(2, '0') + const mm = String(d.getMinutes()).padStart(2, '0') + const ss = String(d.getSeconds()).padStart(2, '0') + const month = String(d.getMonth() + 1).padStart(2, '0') + const day = String(d.getDate()).padStart(2, '0') + return `${month}-${day} ${hh}:${mm}:${ss}` +} + +function formatBytes(n: number): string { + if (n < 1024) return `${n} B` + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB` + return `${(n / (1024 * 1024)).toFixed(2)} MB` +} + +function tryPrettyJson(s: string): string { + try { + return JSON.stringify(JSON.parse(s), null, 2) + } catch { + return s + } +} diff --git a/src/routes/track/-sections/passage-batch-progress.tsx b/src/routes/track/-sections/passage-batch-progress.tsx index b49a3bd..991f214 100644 --- a/src/routes/track/-sections/passage-batch-progress.tsx +++ b/src/routes/track/-sections/passage-batch-progress.tsx @@ -55,17 +55,24 @@ export function PassageBatchProgress({ -
+
+ + {pct}% +
+ aria-hidden="true" + className="h-1.5 w-full overflow-hidden rounded-full bg-[var(--line)]" + > +
+
    diff --git a/src/services/evaluation/kbbi/cari.ts b/src/services/evaluation/kbbi/cari.ts index f327d7f..ca5f4cf 100644 --- a/src/services/evaluation/kbbi/cari.ts +++ b/src/services/evaluation/kbbi/cari.ts @@ -1,3 +1,4 @@ +import { loggedFetch } from '#/services/logs/logged-fetch' import { KBBI_SOURCE_NAMES, KBBI_SOURCES, @@ -79,7 +80,7 @@ export async function cari( fetchInit.proxy = proxy.url fetchInit.dispatcher = proxy.dispatcher } - const res = await fetch(url, fetchInit) + const res = await loggedFetch({ provider: 'kbbi' }, url, fetchInit) if ( source === 'kbbi.kemendikdasmen.go.id' && diff --git a/src/services/evaluation/orchestrator.ts b/src/services/evaluation/orchestrator.ts index 6e75b53..faf7a19 100644 --- a/src/services/evaluation/orchestrator.ts +++ b/src/services/evaluation/orchestrator.ts @@ -13,6 +13,7 @@ import { runKbbiCheck } from '#/services/evaluation/kbbi/checker' import { warmKbbiCaches } from '#/services/evaluation/kbbi/lookup' import { ensureProxyPoolReady } from '#/services/evaluation/kbbi/utils/proxy' import { refreshVocabularyCache } from '#/services/evaluation/vocabulary-cache' +import { withApiLogContext } from '#/services/logs/logged-fetch' const countByCategory = async ( evalJobId: string, @@ -33,6 +34,10 @@ const countByCategory = async ( } export async function runEvaluationAnalysis(evalJobId: string): Promise { + return withApiLogContext({ evalJobId }, () => runEvaluationAnalysisInner(evalJobId)) +} + +async function runEvaluationAnalysisInner(evalJobId: string): Promise { const startedAt = Date.now() await db .update(evaluationJobs) diff --git a/src/services/logs/api-logs.ts b/src/services/logs/api-logs.ts new file mode 100644 index 0000000..f794112 --- /dev/null +++ b/src/services/logs/api-logs.ts @@ -0,0 +1,118 @@ +import { createServerFn } from '@tanstack/react-start' +import { and, desc, eq, inArray, lt, or } from 'drizzle-orm' +import { z } from 'zod' +import { db } from '#/db' +import { apiCallLogs } from '#/db/schema' +import { API_PROVIDERS } from '#/services/logs/logged-fetch' + +const outcomeFilterSchema = z.enum(['all', 'errors', 'success']) + +const cursorSchema = z.object({ + createdAt: z.string().datetime(), + id: z.number().int().positive(), +}) + +const listInputSchema = z.object({ + provider: z.array(z.enum(API_PROVIDERS)).optional(), + outcome: outcomeFilterSchema.default('all'), + trackJobId: z.string().uuid().optional(), + evalJobId: z.string().uuid().optional(), + limit: z.number().int().min(1).max(200).default(50), + cursor: cursorSchema.optional(), +}) + +export const listApiCallLogs = createServerFn({ method: 'GET' }) + .inputValidator(listInputSchema) + .handler(async ({ data }) => { + const conditions = [] + + if (data.provider && data.provider.length > 0) { + conditions.push(inArray(apiCallLogs.provider, data.provider)) + } + + if (data.outcome === 'errors') { + conditions.push( + inArray(apiCallLogs.outcome, ['http_error', 'network_error', 'timeout']), + ) + } else if (data.outcome === 'success') { + conditions.push(eq(apiCallLogs.outcome, 'success')) + } + + if (data.trackJobId) { + conditions.push(eq(apiCallLogs.trackJobId, data.trackJobId)) + } + if (data.evalJobId) { + conditions.push(eq(apiCallLogs.evalJobId, data.evalJobId)) + } + if (data.cursor) { + // Keyset cursor over the composite sort key (createdAt DESC, id DESC). + // Without the secondary id check, rows sharing a createdAt timestamp + // (common under concurrent inserts) can skip or duplicate across pages. + const cursorTime = new Date(data.cursor.createdAt) + conditions.push( + or( + lt(apiCallLogs.createdAt, cursorTime), + and( + eq(apiCallLogs.createdAt, cursorTime), + lt(apiCallLogs.id, data.cursor.id), + ), + )!, + ) + } + + const where = conditions.length > 0 ? and(...conditions) : undefined + + const rows = await db + .select({ + id: apiCallLogs.id, + createdAt: apiCallLogs.createdAt, + provider: apiCallLogs.provider, + method: apiCallLogs.method, + url: apiCallLogs.url, + status: apiCallLogs.status, + outcome: apiCallLogs.outcome, + durationMs: apiCallLogs.durationMs, + errorMessage: apiCallLogs.errorMessage, + bodySizeBytes: apiCallLogs.bodySizeBytes, + bodyTruncated: apiCallLogs.bodyTruncated, + trackJobId: apiCallLogs.trackJobId, + evalJobId: apiCallLogs.evalJobId, + }) + .from(apiCallLogs) + .where(where) + .orderBy(desc(apiCallLogs.createdAt), desc(apiCallLogs.id)) + .limit(data.limit + 1) + + const hasMore = rows.length > data.limit + const page = hasMore ? rows.slice(0, data.limit) : rows + const last = page[page.length - 1] + const nextCursor = + hasMore && last + ? { createdAt: last.createdAt.toISOString(), id: last.id } + : null + + return { rows: page, nextCursor } + }) + +const getInputSchema = z.object({ + id: z.number().int().positive(), +}) + +export const getApiCallLog = createServerFn({ method: 'GET' }) + .inputValidator(getInputSchema) + .handler(async ({ data }) => { + const [row] = await db + .select() + .from(apiCallLogs) + .where(eq(apiCallLogs.id, data.id)) + .limit(1) + return row ?? null + }) + +export type ApiCallLogRow = NonNullable< + Awaited> +> + +export type ApiCallLogListRow = Awaited< + ReturnType +>['rows'][number] diff --git a/src/services/logs/logged-fetch.ts b/src/services/logs/logged-fetch.ts new file mode 100644 index 0000000..ea2e801 --- /dev/null +++ b/src/services/logs/logged-fetch.ts @@ -0,0 +1,250 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import { db } from '#/db' +import { apiCallLogs } from '#/db/schema' +import { getErrorMessage } from '#/lib/utils' + +interface ApiLogStore { + trackJobId?: string + evalJobId?: string +} + +const apiLogStorage = new AsyncLocalStorage() + +export function withApiLogContext( + store: ApiLogStore, + fn: () => Promise, +): Promise { + return apiLogStorage.run(store, fn) +} + +export const API_PROVIDERS = [ + 'openalex', + 'crossref', + 'unpaywall', + 'semantic-scholar', + 'europepmc', + 'pubmed', + 'arxiv', + 'core', + 'doi', + 'kbbi', + 'pdf-download', +] as const + +export type ApiProvider = (typeof API_PROVIDERS)[number] + +export type ApiCallOutcome = + | 'success' + | 'http_error' + | 'network_error' + | 'timeout' + +export interface LogContext { + provider: ApiProvider + trackJobId?: string | null + evalJobId?: string | null + /** Skip body capture entirely — use for binary downloads. */ + metadataOnly?: boolean +} + +const SUCCESS_PREVIEW_BYTES = 2 * 1024 +const ERROR_PREVIEW_BYTES = 1024 * 1024 + +const RELEVANT_HEADERS = new Set([ + 'content-type', + 'content-length', + 'retry-after', +]) +const RATELIMIT_PREFIX = 'x-ratelimit-' + +function pickRelevantHeaders(headers: Headers): Record { + const out: Record = {} + for (const [key, value] of headers.entries()) { + const lc = key.toLowerCase() + if (RELEVANT_HEADERS.has(lc) || lc.startsWith(RATELIMIT_PREFIX)) { + out[lc] = value + } + } + return out +} + +interface LogRow { + ctx: LogContext + url: string + method: string + durationMs: number + status: number | null + outcome: ApiCallOutcome + errorMessage: string | null + responseHeaders: Record | null + bodyPreview: string | null + bodyTruncated: boolean + bodySizeBytes: number | null +} + +function writeLog(row: LogRow): void { + const inherited = apiLogStorage.getStore() + const trackJobId = row.ctx.trackJobId ?? inherited?.trackJobId ?? null + const evalJobId = row.ctx.evalJobId ?? inherited?.evalJobId ?? null + void db + .insert(apiCallLogs) + .values({ + trackJobId, + evalJobId, + provider: row.ctx.provider, + method: row.method, + url: row.url, + status: row.status, + responseHeaders: row.responseHeaders, + bodyPreview: row.bodyPreview, + bodyTruncated: row.bodyTruncated, + bodySizeBytes: row.bodySizeBytes, + outcome: row.outcome, + errorMessage: row.errorMessage, + durationMs: row.durationMs, + }) + .catch(() => { + // Logging must never fail the caller — swallow DB errors silently. + }) +} + +async function readBodyWithCap( + res: Response, + cap: number, +): Promise<{ text: string; size: number; truncated: boolean }> { + const reader = res.body?.getReader() + if (!reader) { + return { text: '', size: 0, truncated: false } + } + + const decoder = new TextDecoder('utf-8', { fatal: false }) + let text = '' + let totalBytes = 0 + let truncated = false + + while (true) { + const { done, value } = await reader.read() + if (done) break + + totalBytes += value.byteLength + + if (text.length < cap) { + const remainingChars = cap - text.length + const chunkText = decoder.decode(value, { stream: true }) + if (chunkText.length <= remainingChars) { + text += chunkText + } else { + text += chunkText.slice(0, remainingChars) + truncated = true + await reader.cancel().catch(() => {}) + break + } + } else { + truncated = true + await reader.cancel().catch(() => {}) + break + } + } + + // Flush any pending multi-byte sequence from the decoder. + if (!truncated) { + text += decoder.decode() + } + + return { text, size: totalBytes, truncated } +} + +export async function loggedFetch( + ctx: LogContext, + url: string, + init?: RequestInit, +): Promise { + const start = Date.now() + const method = init?.method ?? 'GET' + + let res: Response + try { + res = await fetch(url, init) + } catch (err) { + const durationMs = Date.now() - start + const isTimeout = + err instanceof Error && + (err.name === 'TimeoutError' || err.name === 'AbortError') + writeLog({ + ctx, + url, + method, + durationMs, + status: null, + outcome: isTimeout ? 'timeout' : 'network_error', + errorMessage: getErrorMessage(err, 'fetch failed'), + responseHeaders: null, + bodyPreview: null, + bodyTruncated: false, + bodySizeBytes: null, + }) + throw err + } + + const durationMs = Date.now() - start + const headers = pickRelevantHeaders(res.headers) + const outcome: ApiCallOutcome = res.ok ? 'success' : 'http_error' + + if (ctx.metadataOnly) { + const contentLengthHeader = res.headers.get('content-length') + const bodySizeBytes = contentLengthHeader + ? Number.parseInt(contentLengthHeader, 10) + : null + writeLog({ + ctx, + url, + method, + durationMs, + status: res.status, + outcome, + errorMessage: null, + responseHeaders: headers, + bodyPreview: null, + bodyTruncated: false, + bodySizeBytes: Number.isFinite(bodySizeBytes) ? bodySizeBytes : null, + }) + return res + } + + const cap = res.ok ? SUCCESS_PREVIEW_BYTES : ERROR_PREVIEW_BYTES + const clone = res.clone() + + void readBodyWithCap(clone, cap) + .then(({ text, size, truncated }) => { + writeLog({ + ctx, + url, + method, + durationMs, + status: res.status, + outcome, + errorMessage: null, + responseHeaders: headers, + bodyPreview: text, + bodyTruncated: truncated, + bodySizeBytes: size, + }) + }) + .catch((err: unknown) => { + writeLog({ + ctx, + url, + method, + durationMs, + status: res.status, + outcome, + errorMessage: getErrorMessage(err, 'body read failed during logging'), + responseHeaders: headers, + bodyPreview: null, + bodyTruncated: false, + bodySizeBytes: null, + }) + }) + + return res +} diff --git a/src/services/pdf/auto-fetch.ts b/src/services/pdf/auto-fetch.ts index be2025f..bfd4ba4 100644 --- a/src/services/pdf/auto-fetch.ts +++ b/src/services/pdf/auto-fetch.ts @@ -9,6 +9,10 @@ import { paths } from '#/lib/paths' import { getErrorMessage } from '#/lib/utils' import { extractPdfText } from '#/services/pdf/extractor' import { findPdfDiagnostic } from '#/services/pdf/finder' +import { + loggedFetch, + withApiLogContext, +} from '#/services/logs/logged-fetch' const jobIdSchema = z.object({ jobId: z.string().uuid() }) @@ -115,10 +119,14 @@ async function tryDownloadAndExtract( ): Promise { let res: Response try { - res = await fetch(url, { - signal: AbortSignal.timeout(downloadTimeoutMs), - redirect: 'follow', - }) + res = await loggedFetch( + { provider: 'pdf-download', metadataOnly: true }, + url, + { + signal: AbortSignal.timeout(downloadTimeoutMs), + redirect: 'follow', + }, + ) } catch (err) { const raw = getErrorMessage(err, 'Download failed') return { ok: false, error: humanizeFetchError(raw, err) } @@ -160,6 +168,21 @@ async function processReference( author: string }, downloadTimeoutMs: number, +): Promise { + return withApiLogContext({ trackJobId: jobId }, () => + processReferenceInner(jobId, ref, downloadTimeoutMs), + ) +} + +async function processReferenceInner( + jobId: string, + ref: { + id: number + doi: string | null + title: string + author: string + }, + downloadTimeoutMs: number, ): Promise { const [row] = await db .insert(sourcePdfs) diff --git a/src/services/pdf/finder.ts b/src/services/pdf/finder.ts index fe0011c..2628089 100644 --- a/src/services/pdf/finder.ts +++ b/src/services/pdf/finder.ts @@ -11,17 +11,22 @@ import { pubMedEsearchSchema, } from '#/schemas/pdf-finder' import { env } from '#/env' +import { loggedFetch } from '#/services/logs/logged-fetch' const ARXIV_DOI_RE = /^10\.48550\/arxiv\.(.+)$/i const ARXIV_ID_RE = /\/abs\/([^/\s]+)$/ async function tryDoi(doi: string): Promise { try { - const res = await fetch(`https://doi.org/${doi}`, { - redirect: 'follow', - headers: { Accept: 'application/pdf' }, - signal: AbortSignal.timeout(10000), - }) + const res = await loggedFetch( + { provider: 'doi' }, + `https://doi.org/${doi}`, + { + redirect: 'follow', + headers: { Accept: 'application/pdf' }, + signal: AbortSignal.timeout(10000), + }, + ) if ( res.ok && @@ -39,7 +44,8 @@ async function tryDoi(doi: string): Promise { async function tryUnpaywall(doi: string): Promise { if (!env.UNPAYWALL_EMAIL) return null try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'unpaywall' }, `https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=${env.UNPAYWALL_EMAIL}`, { signal: AbortSignal.timeout(10000) }, ) @@ -69,7 +75,8 @@ async function trySemanticScholar( if (env.SEMANTIC_SCHOLAR_API_KEY) { headers['x-api-key'] = env.SEMANTIC_SCHOLAR_API_KEY } - const res = await fetch( + const res = await loggedFetch( + { provider: 'semantic-scholar' }, `https://api.semanticscholar.org/graph/v1/paper/search?query=${query}&limit=3&fields=title,isOpenAccess,openAccessPdf`, { signal: AbortSignal.timeout(10000), headers }, ) @@ -104,7 +111,8 @@ async function trySemanticScholar( async function tryCrossRef(doi: string): Promise { try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'crossref' }, `https://api.crossref.org/works/${encodeURIComponent(doi)}`, { headers: { Accept: 'application/json' }, @@ -142,7 +150,8 @@ function extractOpenAlexUrl( async function tryOpenAlexDoi(doi: string): Promise { try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'openalex' }, `https://api.openalex.org/works/doi:${encodeURIComponent(doi)}`, { headers: { Accept: 'application/json' }, @@ -163,7 +172,8 @@ async function tryOpenAlexDoi(doi: string): Promise { async function tryOpenAlexTitle(title: string): Promise { try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'openalex' }, `https://api.openalex.org/works?search=${encodeURIComponent(title)}&per_page=3`, { headers: { Accept: 'application/json' }, @@ -199,7 +209,8 @@ function europePmcPdf( async function tryEuropePmcDoi(doi: string): Promise { try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'europepmc' }, `https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:${encodeURIComponent(doi)}&format=json&resultType=core`, { signal: AbortSignal.timeout(10000) }, ) @@ -216,7 +227,8 @@ async function tryEuropePmcDoi(doi: string): Promise { async function tryEuropePmcTitle(title: string): Promise { try { const q = `TITLE:"${title.replaceAll('"', '')}"` - const res = await fetch( + const res = await loggedFetch( + { provider: 'europepmc' }, `https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${encodeURIComponent(q)}&format=json&resultType=core`, { signal: AbortSignal.timeout(10000) }, ) @@ -234,7 +246,8 @@ async function fetchPubMedPmcid(term: string): Promise { const apiKey = env.NCBI_API_KEY const keyParam = apiKey ? `&api_key=${encodeURIComponent(apiKey)}` : '' try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'pubmed' }, `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pmc&term=${encodeURIComponent(term)}&retmode=json&retmax=1${keyParam}`, { signal: AbortSignal.timeout(10000) }, ) @@ -279,7 +292,8 @@ async function tryArxiv( try { const query = `ti:"${title.replaceAll('"', '')}"` - const res = await fetch( + const res = await loggedFetch( + { provider: 'arxiv' }, `https://export.arxiv.org/api/query?search_query=${encodeURIComponent(query)}&max_results=1`, { signal: AbortSignal.timeout(10000) }, ) @@ -305,7 +319,8 @@ async function tryCoreAc(title: string): Promise { if (!apiKey) return null try { - const res = await fetch( + const res = await loggedFetch( + { provider: 'core' }, `https://api.core.ac.uk/v3/search/works?q=${encodeURIComponent(title)}&limit=3`, { headers: { diff --git a/src/services/purge.ts b/src/services/purge.ts index aec6056..a02f5d1 100644 --- a/src/services/purge.ts +++ b/src/services/purge.ts @@ -1,10 +1,10 @@ import { readdir, rm, stat } from 'node:fs/promises' import { join } from 'node:path' -import { and, inArray, lt } from 'drizzle-orm' +import { and, inArray, isNull, lt } from 'drizzle-orm' import { createServerFn } from '@tanstack/react-start' import { z } from 'zod' import { db } from '#/db' -import { evaluationJobs, jobs, sourcePdfs } from '#/db/schema' +import { apiCallLogs, evaluationJobs, jobs, sourcePdfs } from '#/db/schema' import { assertLocalOnly } from '#/env' import { paths } from '#/lib/paths' import { getConfig } from '#/services/configurations-cache' @@ -17,6 +17,7 @@ export type PurgeResult = { bytesFreed: number orphanFilesDeleted: number orphanBytesFreed: number + apiLogsDeleted: number } export type PruneAllResult = { @@ -223,6 +224,19 @@ export const purgeHistory = createServerFn({ method: 'POST' }).handler( }, ) + // API logs attached to a job already cascade-deleted with the jobs above. + // Sweep unattached logs (both FKs null) older than the retention window. + const deletedApiLogs = await db + .delete(apiCallLogs) + .where( + and( + isNull(apiCallLogs.trackJobId), + isNull(apiCallLogs.evalJobId), + lt(apiCallLogs.createdAt, retentionCutoff), + ), + ) + .returning({ id: apiCallLogs.id }) + return { trackJobsDeleted: trackJobIds.length, evaluationJobsDeleted: evalJobIds.length, @@ -233,6 +247,7 @@ export const purgeHistory = createServerFn({ method: 'POST' }).handler( userSweep.count + sourceSweep.count + evalSweep.count, orphanBytesFreed: userSweep.bytes + sourceSweep.bytes + evalSweep.bytes, + apiLogsDeleted: deletedApiLogs.length, } }, ) @@ -284,13 +299,15 @@ export const pruneAll = createServerFn({ method: 'POST' }) bytesFreed += trackSweep.bytes + sourceSweep.bytes + evalSweep.bytes // ON DELETE CASCADE on jobs/evaluation_jobs takes care of citations, - // references, matches, pages, findings, summaries, etc. + // references, matches, pages, findings, summaries, and attached api logs. if (allTrackJobs.length > 0) { await db.delete(jobs) } if (allEvalJobs.length > 0) { await db.delete(evaluationJobs) } + // Unattached api logs survive cascade — wipe them too. + await db.delete(apiCallLogs) return { trackJobsDeleted: allTrackJobs.length,