[pull] main from firecrawl:main

apps/api/openapi.json

@@ -2277,6 +2277,7 @@
                 "markdown",
                 "html",
                 "rawHtml",
+                "rawBase64",
                 "links",
                 "screenshot",
                 "screenshot@fullPage",
@@ -2285,7 +2286,7 @@
                 "branding"
               ]
             },
-            "description": "Formats to include in the output.",
+            "description": "Formats to include in the output. `rawBase64` must be requested by itself.",
             "default": ["markdown"]
           },
           "onlyMainContent": {
@@ -2606,6 +2607,11 @@
                 "nullable": true,
                 "description": "Raw HTML content of the page if `rawHtml` is in `formats`"
               },
+              "rawBase64": {
+                "type": "string",
+                "nullable": true,
+                "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+              },
               "screenshot": {
                 "type": "string",
                 "nullable": true,
@@ -2866,6 +2872,11 @@
                   "nullable": true,
                   "description": "Raw HTML content of the page if `includeRawHtml`  is true"
                 },
+                "rawBase64": {
+                  "type": "string",
+                  "nullable": true,
+                  "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+                },
                 "links": {
                   "type": "array",
                   "items": {
@@ -3003,6 +3014,11 @@
                   "nullable": true,
                   "description": "Raw HTML content of the page if `includeRawHtml`  is true"
                 },
+                "rawBase64": {
+                  "type": "string",
+                  "nullable": true,
+                  "description": "Base64-encoded original response body if `rawBase64` is in `formats`"
+                },
                 "links": {
                   "type": "array",
                   "items": {

apps/api/src/__tests__/snips/mocks/raw-base64.json

@@ -0,0 +1,70 @@
+[
+  {
+    "time": 1,
+    "options": {
+      "url": "<fire-engine>/scrape",
+      "method": "POST",
+      "body": {
+        "url": "https://example.com/raw",
+        "engine": "chrome-cdp",
+        "format": "rawBase64"
+      }
+    },
+    "result": {
+      "status": 200,
+      "headers": {},
+      "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"text/html; charset=utf-8\"},\"file\":{\"name\":\"raw.html\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"
+    }
+  },
+  {
+    "time": 2,
+    "options": {
+      "url": "<fire-engine>/scrape",
+      "method": "POST",
+      "body": {
+        "url": "https://example.com/raw.pdf",
+        "engine": "chrome-cdp",
+        "format": "rawBase64"
+      }
+    },
+    "result": {
+      "status": 200,
+      "headers": {},
+      "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw.pdf\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"application/pdf\"},\"file\":{\"name\":\"raw.pdf\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"
+    }
+  },
+  {
+    "time": 3,
+    "options": {
+      "url": "<fire-engine>/scrape",
+      "method": "POST",
+      "body": {
+        "url": "https://example.com/raw.docx",
+        "engine": "chrome-cdp",
+        "format": "rawBase64"
+      }
+    },
+    "result": {
+      "status": 200,
+      "headers": {},
+      "body": "{\"timeTaken\":0.1,\"content\":\"\",\"url\":\"https://example.com/raw.docx\",\"pageStatusCode\":200,\"responseHeaders\":{\"content-type\":\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\"},\"file\":{\"name\":\"raw.docx\",\"content\":\"PGh0bWw+cmF3PC9odG1sPg==\"}}"
+    }
+  },
+  {
+    "time": 4,
+    "options": {
+      "url": "<fire-engine>/scrape",
+      "method": "POST",
+      "body": {
+        "url": "https://example.com/raw-error",
+        "engine": "chrome-cdp",
+        "format": "rawBase64"
+      }
+    },
+    "result": {
+      "status": 200,
+      "headers": {},
+      "body": "{\"timeTaken\":0.1,\"content\":\"Not found\",\"url\":\"https://example.com/raw-error\",\"pageStatusCode\":404,\"responseHeaders\":{\"content-type\":\"text/html; charset=utf-8\"}}"
+    }
+  }
+]

apps/api/src/__tests__/snips/v1/types-validation.test.ts

@@ -59,6 +59,22 @@ describe("V1 Types Validation", () => {
       expect(result.timeout).toBe(60000);
     });
 
+    it("should only allow rawBase64 as the sole format", () => {
+      expect(
+        scrapeRequestSchema.parse({
+          url: "https://example.com/file",
+          formats: ["rawBase64"],
+        }).formats,
+      ).toEqual(["rawBase64"]);
+
+      expect(() =>
+        scrapeRequestSchema.parse({
+          url: "https://example.com/file",
+          formats: ["markdown", "rawBase64"],
+        }),
+      ).toThrow("The rawBase64 format cannot be combined with other formats");
+    });
+
     it("should reject invalid URL", () => {
       const input = {
         url: "not-a-url",

apps/api/src/__tests__/snips/v2/types-validation.test.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import {
   scrapeRequestSchema,
+  parseRequestSchema,
   scrapeOptions,
   extractRequestSchema,
   crawlRequestSchema,
@@ -83,6 +84,22 @@ describe("V2 Types Validation", () => {
       expect(result.formats).toEqual([{ type: "markdown" }, { type: "html" }]);
     });
 
+    it("should only allow rawBase64 as the sole format", () => {
+      expect(
+        scrapeRequestSchema.parse({
+          url: "https://example.com/file",
+          formats: ["rawBase64"],
+        }).formats,
+      ).toEqual([{ type: "rawBase64" }]);
+
+      expect(() =>
+        scrapeRequestSchema.parse({
+          url: "https://example.com/file",
+          formats: ["markdown", "rawBase64"],
+        }),
+      ).toThrow("The rawBase64 format cannot be combined with other formats");
+    });
+
     it("should accept video format as string and object", () => {
       const stringInput: ScrapeRequestInput = {
         url: "https://example.com",
@@ -685,6 +702,22 @@ describe("V2 Types Validation", () => {
     });
   });
 
+  describe("parseRequestSchema", () => {
+    it("should reject rawBase64 for file uploads", () => {
+      expect(() =>
+        parseRequestSchema.parse({
+          formats: ["rawBase64"],
+          file: {
+            buffer: Buffer.from("raw upload"),
+            filename: "upload.html",
+            contentType: "text/html",
+            kind: "html",
+          },
+        }),
+      ).toThrow("The rawBase64 format is not supported for parse uploads");
+    });
+  });
+
   describe("extractRequestSchema", () => {
     it("should accept valid extract request with urls", () => {
       const input: ExtractRequestInput = {

apps/api/src/controllers/__tests__/auth.test.ts

@@ -51,11 +51,24 @@ vi.mock("../../db/rpc", () => ({
   authCreditUsageChunkFromTeam: vi.fn(),
 }));
 
-vi.mock("../../services/rate-limiter", () => ({
-  getRateLimiter: vi.fn(),
-  getAutumnRateLimiter: vi.fn(),
+// The limiter builders are mocked, but getRateLimitOverride is kept real: it is
+// the single source of truth for override resolution, and auth.ts calls it to
+// decide whether the Autumn multiplier is needed at all. Stub ioredis so
+// importing the real module doesn't open a connection.
+vi.mock("ioredis", () => ({
+  default: class {},
 }));
 
+vi.mock("../../services/rate-limiter", async importOriginal => {
+  const actual =
+    await importOriginal<typeof import("../../services/rate-limiter")>();
+  return {
+    ...actual,
+    getRateLimiter: vi.fn(),
+    getAutumnRateLimiter: vi.fn(),
+  };
+});
+
 vi.mock("../../lib/keyless", async importOriginal => {
   const actual = await importOriginal<typeof import("../../lib/keyless")>();
   return {
@@ -88,6 +101,7 @@ describe("authenticateUser", () => {
     config.MCP_DELEGATED_CREDENTIAL_SECRET;
   const originalIntrospectUrl = config.OAUTH_INTROSPECT_URL;
   const originalIntrospectSecret = config.OAUTH_INTROSPECT_SECRET;
+  const originalPreviewToken = config.PREVIEW_TOKEN;
 
   beforeEach(() => {
     vi.mocked(isKeylessConfigured).mockReturnValue(false);
@@ -105,6 +119,7 @@ describe("authenticateUser", () => {
       originalMcpDelegatedCredentialSecret;
     config.OAUTH_INTROSPECT_URL = originalIntrospectUrl;
     config.OAUTH_INTROSPECT_SECRET = originalIntrospectSecret;
+    config.PREVIEW_TOKEN = originalPreviewToken;
     vi.unstubAllGlobals();
     vi.clearAllMocks();
   });
@@ -497,6 +512,105 @@ describe("authenticateUser", () => {
     });
   });
 
+  it("passes the org rate-limit overrides to the API-key rate limiter", async () => {
+    config.USE_DB_AUTHENTICATION = true;
+    vi.mocked(getValue).mockResolvedValue(null);
+    const flags = { rateLimitOverrides: { scrape: 42 } };
+    vi.mocked(authCreditUsageChunk).mockResolvedValue([
+      {
+        api_key: "00000000-0000-4000-8000-000000000000",
+        api_key_id: 1,
+        team_id: "team-1",
+        org_id: "org-1",
+        flags,
+      },
+    ]);
+    vi.mocked(redlock.using).mockImplementation(
+      async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),
+    );
+    vi.mocked(autumnService.getRateLimitMultiplier).mockResolvedValue(50);
+
+    const auth = await authenticateUser(
+      {
+        headers: {
+          authorization: "Bearer 00000000-0000-4000-8000-000000000000",
+        },
+        socket: { remoteAddress: "127.0.0.1" },
+      },
+      {},
+      RateLimiterMode.Scrape,
+    );
+
+    expect(auth.success).toBe(true);
+    // The override replaces the whole base × multiplier computation, so the
+    // Autumn multiplier is never fetched and a neutral 1 is passed instead.
+    expect(autumnService.getRateLimitMultiplier).not.toHaveBeenCalled();
+    expect(getAutumnRateLimiter).toHaveBeenCalledWith(
+      RateLimiterMode.Scrape,
+      1,
+      flags,
+    );
+  });
+
+  it("still fetches the Autumn multiplier when no override covers the mode", async () => {
+    config.USE_DB_AUTHENTICATION = true;
+    vi.mocked(getValue).mockResolvedValue(null);
+    const flags = { rateLimitOverrides: { crawl: 42 } };
+    vi.mocked(authCreditUsageChunk).mockResolvedValue([
+      {
+        api_key: "00000000-0000-4000-8000-000000000000",
+        api_key_id: 1,
+        team_id: "team-1",
+        org_id: "org-1",
+        flags,
+      },
+    ]);
+    vi.mocked(redlock.using).mockImplementation(
+      async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),
+    );
+    vi.mocked(autumnService.getRateLimitMultiplier).mockResolvedValue(50);
+
+    const auth = await authenticateUser(
+      {
+        headers: {
+          authorization: "Bearer 00000000-0000-4000-8000-000000000000",
+        },
+        socket: { remoteAddress: "127.0.0.1" },
+      },
+      {},
+      RateLimiterMode.Scrape,
+    );
+
+    expect(auth.success).toBe(true);
+    expect(autumnService.getRateLimitMultiplier).toHaveBeenCalledTimes(1);
+    expect(getAutumnRateLimiter).toHaveBeenCalledWith(
+      RateLimiterMode.Scrape,
+      50,
+      flags,
+    );
+  });
+
+  it("leaves the preview token on the static rate limiter", async () => {
+    config.USE_DB_AUTHENTICATION = true;
+    config.PREVIEW_TOKEN = "preview-token";
+    vi.mocked(getRateLimiter).mockReturnValue({
+      consume: vi.fn().mockResolvedValue(undefined),
+    } as never);
+
+    const auth = await authenticateUser(
+      {
+        headers: { authorization: "Bearer preview-token" },
+        socket: { remoteAddress: "127.0.0.1" },
+      },
+      {},
+      RateLimiterMode.Scrape,
+    );
+
+    expect(auth.success).toBe(true);
+    expect(getRateLimiter).toHaveBeenCalledWith(RateLimiterMode.Preview);
+    expect(getAutumnRateLimiter).not.toHaveBeenCalled();
+  });
+
   it("clears purpose-qualified and legacy ACUC cache entries", async () => {
     await clearACUC("api-key");