From 9cbddcec564c60a8d1b79ffa5b90f7d12335caef Mon Sep 17 00:00:00 2001 From: Prasanna721 <106952318+Prasanna721@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:46:21 +0000 Subject: [PATCH 1/2] docs: historical backfill guide (#1474) Adds a focused guide for backfilling dated documents with `documentDate` and the batch ingestion API. - includes TypeScript and Python batch examples plus optional completion polling - links the guide from the docs navigation and ingestion entry points Validated with `bunx mintlify@latest validate` and `bunx mintlify@latest broken-links`. --- apps/docs/docs.json | 5 + apps/docs/ingestion/add-memories.mdx | 1 + .../batch-ingest-historical-data.mdx | 145 ++++++++++++++++++ apps/docs/using-supermemory.mdx | 1 + 4 files changed, 152 insertions(+) create mode 100644 apps/docs/ingestion/batch-ingest-historical-data.mdx diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 9400ae9f6..3891a43bd 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -194,6 +194,11 @@ { "group": "Other resources", "pages": [ + { + "group": "General", + "icon": "book-open", + "pages": ["ingestion/batch-ingest-historical-data"] + }, { "group": "Benchmarking", "icon": "flask-conical", diff --git a/apps/docs/ingestion/add-memories.mdx b/apps/docs/ingestion/add-memories.mdx index 39d7c1035..ab19a5056 100644 --- a/apps/docs/ingestion/add-memories.mdx +++ b/apps/docs/ingestion/add-memories.mdx @@ -496,6 +496,7 @@ console.log(doc.status); // "queued" | "processing" | "done" ## Next Steps +- [How to backfill historical data](/ingestion/batch-ingest-historical-data) — Import dated content with the batch API - [Search Memories](/recall/search) — Query your content - [User Profiles](/recall/user-profiles) — Get user context - [Organizing & Filtering](/concepts/filtering) — Container tags and metadata diff --git a/apps/docs/ingestion/batch-ingest-historical-data.mdx b/apps/docs/ingestion/batch-ingest-historical-data.mdx new file mode 100644 index 000000000..17ff4e1bd --- /dev/null +++ b/apps/docs/ingestion/batch-ingest-historical-data.mdx @@ -0,0 +1,145 @@ +--- +title: "How to backfill historical data into Supermemory" +sidebarTitle: "Backfill historical data" +description: "Backfill historical documents into Supermemory with documentDate, stable custom IDs, and the batch ingestion API." +icon: "history" +--- + +Use `POST /v3/documents/batch` to backfill exports, emails, messages, or other dated records. + + + Sort the source data oldest to newest, add `documentDate` to every document. + + +## Backfill in batches + +Backfill dated content by setting `documentDate` on each document, sorting the source records oldest to newest, and sending them in batches. Each request can contain up to 600 documents. + +**Endpoint:** [`POST /v3/documents/batch`](/api-reference/ingest/batch-add-documents) + + + +```typescript TypeScript +import Supermemory from "supermemory"; + +type SourceDocument = { + id: string; + content: string; + createdAt: string; +}; + +const client = new Supermemory(); +const batchSize = 100; + +async function backfillHistoricalData(sourceDocuments: SourceDocument[]) { + const documents = sourceDocuments + .map((document) => ({ + content: document.content, + customId: document.id, + documentDate: new Date(document.createdAt).toISOString() + })) + .sort((a, b) => a.documentDate.localeCompare(b.documentDate)); + + for (let offset = 0; offset < documents.length; offset += batchSize) { + const result = await client.documents.batchAdd({ + containerTag: "historical_import", + documents: documents.slice(offset, offset + batchSize) + }); + + if (result.failed > 0) { + throw new Error(`${result.failed} documents failed to ingest`); + } + } +} +``` + +```python Python +from datetime import datetime, timezone +from supermemory import Supermemory + +client = Supermemory() +batch_size = 100 + +def to_utc(value: str) -> str: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("created_at must include a timezone") + return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + +def backfill_historical_data(source_documents: list[dict[str, str]]) -> None: + documents = sorted( + [ + { + "content": document["content"], + "custom_id": document["id"], + "document_date": to_utc(document["created_at"]), + } + for document in source_documents + ], + key=lambda document: document["document_date"], + ) + + for offset in range(0, len(documents), batch_size): + result = client.documents.batch_add( + container_tag="historical_import", + documents=documents[offset : offset + batch_size], + ) + + if result.failed > 0: + raise RuntimeError(f"{result.failed} documents failed to ingest") +``` + + + +## Optional: wait for processing to finish + +**Endpoint:** [`GET /v3/documents/{id}`](/api-reference/documents/get-document) + +The batch endpoint returns after accepting the documents. If a later step depends on completed memory generation, poll the returned document IDs until both `status` and `dreamingStatus` are `done`. + + + +```typescript TypeScript +async function waitUntilDone(ids: string[]) { + while (true) { + const documents = await Promise.all( + ids.map((id) => client.documents.get(id)) + ); + + if (documents.some((document) => document.status === "failed")) { + throw new Error("A document failed to process"); + } + + if ( + documents.every( + (document) => + document.status === "done" && document.dreamingStatus === "done" + ) + ) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } +} +``` + +```python Python +import time + +def wait_until_done(ids: list[str]) -> None: + while True: + documents = [client.documents.get(document_id) for document_id in ids] + + if any(document.status == "failed" for document in documents): + raise RuntimeError("A document failed to process") + + if all( + document.status == "done" and document.dreaming_status == "done" + for document in documents + ): + return + + time.sleep(10) +``` + + diff --git a/apps/docs/using-supermemory.mdx b/apps/docs/using-supermemory.mdx index f65ed2e3c..be38bf807 100644 --- a/apps/docs/using-supermemory.mdx +++ b/apps/docs/using-supermemory.mdx @@ -18,6 +18,7 @@ Everything in this section is one of four steps. Same loop whether you're buildi + From 5ecbc263450def05fea29c4adcd30aa85b5af31c Mon Sep 17 00:00:00 2001 From: Dhravya Date: Fri, 14 Aug 2026 22:36:52 +0000 Subject: [PATCH 2/2] fix(mcp): surface real API error messages instead of 'restricted or blocked' (#1406) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why? Plain **T-1554**: a user with a **read-only** MCP OAuth grant got 403s on memory listing, and the client rendered them as *"Access forbidden. Your account may be restricted or blocked."* The API's actual error body said `{"error": "This API key has read-only access"}` — but `handleError` discarded it, so the user (and support) chased a nonexistent account ban. Two masking layers: 1. `handleError` used the raw error `message`, which for our raw-fetch endpoints was a hardcoded string ("Failed to fetch documents") or unparsed JSON, and fell back to the scary "restricted or blocked" text when empty. 2. `getDocuments` didn't read the response body at all. ## What? - New `extractApiErrorMessage()` unwraps JSON error bodies (`{"error": ...}` / `{"message": ...}`) so the API's real reason reaches the user. - `getDocuments` and `listMemoryEntries` now pass the (unwrapped) response body through with the status, letting `handleError` apply status-aware fallbacks when the body is empty. - Reworded the empty-body 403 fallback to point at the common cause first: *"Access forbidden. This connection may be read-only or scoped to specific spaces — reconnect with broader access, or check your account status."* Companion API-side fix (read-only grants couldn't call semantically-read POST list endpoints at all): supermemoryai/mono#2772. ## Testing - Added tests: a 403 with a JSON error body surfaces the API's message; an empty-body 403 gets the scope-aware fallback. `vitest run src/server/client/index.test.ts` — 3 passed. - `tsc --noEmit -p tsconfig.json` clean. (The `check-types` script also runs `tsconfig.widget.json`, which fails on origin/main with a pre-existing `UseAppOptions.strict` error, unrelated.) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- > [!NOTE] > **Low Risk** > User-facing error text only in the MCP client; no auth or API behavior changes. > > **Overview** > **MCP client errors now show what the API actually returned** instead of hardcoded strings or misleading “restricted or blocked” text. > > Adds `extractApiErrorMessage()` to parse JSON bodies (`error` / `message` fields) from failed responses. **`getDocuments`** and **`listMemoryEntries`** read the response body on non-OK status and attach the unwrapped message (with status) for **`handleError`**, which also uses the helper on error messages. When a 403 has no body message, the fallback now points users toward **read-only or scoped OAuth** rather than an account ban. > > Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 1f492470cf4b619e58eea6d45af1dfa0b8cad0c4. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot). --- apps/mcp/src/server/client/index.ts | 30 ++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/apps/mcp/src/server/client/index.ts b/apps/mcp/src/server/client/index.ts index cc45d438b..ad9695a40 100644 --- a/apps/mcp/src/server/client/index.ts +++ b/apps/mcp/src/server/client/index.ts @@ -149,6 +149,19 @@ function objectProperty(value: unknown, key: string): unknown { : undefined } +// API error bodies are JSON like {"error": "..."} — unwrap them so users see +// the real reason instead of raw JSON or a generic fallback. +function extractApiErrorMessage(raw: unknown): string | undefined { + if (typeof raw !== "string" || !raw) return undefined + try { + const parsed = JSON.parse(raw) as { error?: unknown; message?: unknown } + if (typeof parsed.error === "string" && parsed.error) return parsed.error + if (typeof parsed.message === "string" && parsed.message) + return parsed.message + } catch {} + return raw +} + export class SupermemoryClient { private client: Supermemory private containerTag: string @@ -371,7 +384,8 @@ export class SupermemoryClient { signal, }) if (!response.ok) { - throw Object.assign(new Error("Failed to fetch documents"), { + const message = extractApiErrorMessage(await response.text()) + throw Object.assign(new Error(message ?? ""), { status: response.status, }) } @@ -432,11 +446,10 @@ export class SupermemoryClient { }) if (!response.ok) { - const message = await response.text() - throw Object.assign( - new Error(message || "Failed to fetch memory entries"), - { status: response.status }, - ) + const message = extractApiErrorMessage(await response.text()) + throw Object.assign(new Error(message ?? ""), { + status: response.status, + }) } return memoryEntriesResponseSchema.parse(await response.json()) @@ -466,8 +479,7 @@ export class SupermemoryClient { const status = objectProperty(error, "status") if (typeof status === "number") { - const rawMessage = objectProperty(error, "message") - const message = typeof rawMessage === "string" ? rawMessage : undefined + const message = extractApiErrorMessage(objectProperty(error, "message")) switch (status) { case 400: case 422: @@ -479,7 +491,7 @@ export class SupermemoryClient { case 403: throw new Error( message || - "Access forbidden. Your account may be restricted or blocked.", + "Access forbidden. This connection may be read-only or scoped to specific spaces — reconnect with broader access, or check your account status.", ) case 404: throw new Error("Not found.")