diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 9400ae9f6..3891a43bd 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -194,6 +194,11 @@ { "group": "Other resources", "pages": [ + { + "group": "General", + "icon": "book-open", + "pages": ["ingestion/batch-ingest-historical-data"] + }, { "group": "Benchmarking", "icon": "flask-conical", diff --git a/apps/docs/ingestion/add-memories.mdx b/apps/docs/ingestion/add-memories.mdx index 39d7c1035..ab19a5056 100644 --- a/apps/docs/ingestion/add-memories.mdx +++ b/apps/docs/ingestion/add-memories.mdx @@ -496,6 +496,7 @@ console.log(doc.status); // "queued" | "processing" | "done" ## Next Steps +- [How to backfill historical data](/ingestion/batch-ingest-historical-data) — Import dated content with the batch API - [Search Memories](/recall/search) — Query your content - [User Profiles](/recall/user-profiles) — Get user context - [Organizing & Filtering](/concepts/filtering) — Container tags and metadata diff --git a/apps/docs/ingestion/batch-ingest-historical-data.mdx b/apps/docs/ingestion/batch-ingest-historical-data.mdx new file mode 100644 index 000000000..17ff4e1bd --- /dev/null +++ b/apps/docs/ingestion/batch-ingest-historical-data.mdx @@ -0,0 +1,145 @@ +--- +title: "How to backfill historical data into Supermemory" +sidebarTitle: "Backfill historical data" +description: "Backfill historical documents into Supermemory with documentDate, stable custom IDs, and the batch ingestion API." +icon: "history" +--- + +Use `POST /v3/documents/batch` to backfill exports, emails, messages, or other dated records. + + + Sort the source data oldest to newest, add `documentDate` to every document. + + +## Backfill in batches + +Backfill dated content by setting `documentDate` on each document, sorting the source records oldest to newest, and sending them in batches. Each request can contain up to 600 documents. + +**Endpoint:** [`POST /v3/documents/batch`](/api-reference/ingest/batch-add-documents) + + + +```typescript TypeScript +import Supermemory from "supermemory"; + +type SourceDocument = { + id: string; + content: string; + createdAt: string; +}; + +const client = new Supermemory(); +const batchSize = 100; + +async function backfillHistoricalData(sourceDocuments: SourceDocument[]) { + const documents = sourceDocuments + .map((document) => ({ + content: document.content, + customId: document.id, + documentDate: new Date(document.createdAt).toISOString() + })) + .sort((a, b) => a.documentDate.localeCompare(b.documentDate)); + + for (let offset = 0; offset < documents.length; offset += batchSize) { + const result = await client.documents.batchAdd({ + containerTag: "historical_import", + documents: documents.slice(offset, offset + batchSize) + }); + + if (result.failed > 0) { + throw new Error(`${result.failed} documents failed to ingest`); + } + } +} +``` + +```python Python +from datetime import datetime, timezone +from supermemory import Supermemory + +client = Supermemory() +batch_size = 100 + +def to_utc(value: str) -> str: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("created_at must include a timezone") + return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + +def backfill_historical_data(source_documents: list[dict[str, str]]) -> None: + documents = sorted( + [ + { + "content": document["content"], + "custom_id": document["id"], + "document_date": to_utc(document["created_at"]), + } + for document in source_documents + ], + key=lambda document: document["document_date"], + ) + + for offset in range(0, len(documents), batch_size): + result = client.documents.batch_add( + container_tag="historical_import", + documents=documents[offset : offset + batch_size], + ) + + if result.failed > 0: + raise RuntimeError(f"{result.failed} documents failed to ingest") +``` + + + +## Optional: wait for processing to finish + +**Endpoint:** [`GET /v3/documents/{id}`](/api-reference/documents/get-document) + +The batch endpoint returns after accepting the documents. If a later step depends on completed memory generation, poll the returned document IDs until both `status` and `dreamingStatus` are `done`. + + + +```typescript TypeScript +async function waitUntilDone(ids: string[]) { + while (true) { + const documents = await Promise.all( + ids.map((id) => client.documents.get(id)) + ); + + if (documents.some((document) => document.status === "failed")) { + throw new Error("A document failed to process"); + } + + if ( + documents.every( + (document) => + document.status === "done" && document.dreamingStatus === "done" + ) + ) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } +} +``` + +```python Python +import time + +def wait_until_done(ids: list[str]) -> None: + while True: + documents = [client.documents.get(document_id) for document_id in ids] + + if any(document.status == "failed" for document in documents): + raise RuntimeError("A document failed to process") + + if all( + document.status == "done" and document.dreaming_status == "done" + for document in documents + ): + return + + time.sleep(10) +``` + + diff --git a/apps/docs/using-supermemory.mdx b/apps/docs/using-supermemory.mdx index f65ed2e3c..be38bf807 100644 --- a/apps/docs/using-supermemory.mdx +++ b/apps/docs/using-supermemory.mdx @@ -18,6 +18,7 @@ Everything in this section is one of four steps. Same loop whether you're buildi + diff --git a/apps/mcp/src/server/client/index.ts b/apps/mcp/src/server/client/index.ts index cc45d438b..ad9695a40 100644 --- a/apps/mcp/src/server/client/index.ts +++ b/apps/mcp/src/server/client/index.ts @@ -149,6 +149,19 @@ function objectProperty(value: unknown, key: string): unknown { : undefined } +// API error bodies are JSON like {"error": "..."} — unwrap them so users see +// the real reason instead of raw JSON or a generic fallback. +function extractApiErrorMessage(raw: unknown): string | undefined { + if (typeof raw !== "string" || !raw) return undefined + try { + const parsed = JSON.parse(raw) as { error?: unknown; message?: unknown } + if (typeof parsed.error === "string" && parsed.error) return parsed.error + if (typeof parsed.message === "string" && parsed.message) + return parsed.message + } catch {} + return raw +} + export class SupermemoryClient { private client: Supermemory private containerTag: string @@ -371,7 +384,8 @@ export class SupermemoryClient { signal, }) if (!response.ok) { - throw Object.assign(new Error("Failed to fetch documents"), { + const message = extractApiErrorMessage(await response.text()) + throw Object.assign(new Error(message ?? ""), { status: response.status, }) } @@ -432,11 +446,10 @@ export class SupermemoryClient { }) if (!response.ok) { - const message = await response.text() - throw Object.assign( - new Error(message || "Failed to fetch memory entries"), - { status: response.status }, - ) + const message = extractApiErrorMessage(await response.text()) + throw Object.assign(new Error(message ?? ""), { + status: response.status, + }) } return memoryEntriesResponseSchema.parse(await response.json()) @@ -466,8 +479,7 @@ export class SupermemoryClient { const status = objectProperty(error, "status") if (typeof status === "number") { - const rawMessage = objectProperty(error, "message") - const message = typeof rawMessage === "string" ? rawMessage : undefined + const message = extractApiErrorMessage(objectProperty(error, "message")) switch (status) { case 400: case 422: @@ -479,7 +491,7 @@ export class SupermemoryClient { case 403: throw new Error( message || - "Access forbidden. Your account may be restricted or blocked.", + "Access forbidden. This connection may be read-only or scoped to specific spaces — reconnect with broader access, or check your account status.", ) case 404: throw new Error("Not found.")