diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000000..69609767c2 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,253 @@ +# Plan: Move `createBatch` and dependencies to `@datadog/js-core/transport` + +## Goal + +Move the batching transport stack from `packages/browser-core/src/transport/` to +`packages/js-core/src/transport/` so it becomes available to all SDK consumers +(including non-browser environments) via `@datadog/js-core/transport`. + +--- + +## Dependency Analysis + +### Direct dependencies of `createBatch` (batch.ts) + +| Import | Location | Browser-specific? | Status | +| ------------------------------------------------- | ------------------------------------------------ | ----------------------- | --------------------- | +| `EndpointBuilder` | `@datadog/js-core/transport` | No | ✅ Already in js-core | +| `Encoder`, `createIdentityEncoder` | `browser-core/tools/encoder` | No | Needs move | +| `createHttpRequest`, `Payload` | `browser-core/transport/httpRequest` | Yes (fetch, sendBeacon) | Needs design decision | +| `createFlushController`, `FlushEvent`, … | `browser-core/transport/flushController` | Partially | Needs move | +| `isPageExitReason`, `createPageMayExitObservable` | `browser-core/browser/pageMayExitObservable` | Yes | Needs design decision | +| `Observable` | `browser-core/tools/observable` | No | Needs move | +| `display`, `DOCS_TROUBLESHOOTING`, … | `browser-core/tools/display` | No (wraps js-core/util) | Can stay / re-export | +| `jsonStringify` | `browser-core/tools/serialisation/jsonStringify` | No | Needs move | +| `Context` | `browser-core/tools/serialisation/context` | No (pure types) | Needs move | +| `objectValues` | `browser-core/tools/utils/polyfills` | No | Needs move | +| `computeBytesCount`, `ONE_KIBI_BYTE` | `browser-core/tools/utils/byteUtils` | No | Needs move | +| `mockable` | `browser-core/tools/mockable` | No | Needs move | + +### Transitive dependencies + +**`flushController.ts`** needs: + +- `Observable` (browser-core) +- `setTimeout` / `clearTimeout` from `timer.ts` (browser-core, wraps globalObject for ZoneJS) +- `pageMayExitObservable` types — already injected as **parameter** ✓ +- `RECOMMENDED_REQUEST_BYTES_LIMIT` from `httpRequest.ts` (circular if we move both) + +**`httpRequest.ts`** needs: + +- `Observable` (browser-core) +- `fetch.ts` (browser-core, uses `globalObject.fetch` + ZoneJS patching) +- `sendWithRetryStrategy.ts` (browser-core) +- `byteUtils` (browser-core) + +**`sendWithRetryStrategy.ts`** needs: + +- `Observable` (browser-core) +- `timer.ts` (`setTimeout`) (browser-core) +- `byteUtils`, `responseUtils` (browser-core) + +**`observable.ts`** needs: + +- `queueMicrotask.ts` (browser-core) → needs `monitor` + `globalObject` + +**`timer.ts`** needs: + +- `globalObject` (`js-core/util` ✓) +- `getZoneJsOriginalValue` (browser-core) +- `monitor` (browser-core wrapper around `js-core/monitor`) + +--- + +## Design Decisions Required + +### 1. How to handle `pageMayExitObservable` (browser-specific) + +`flushController` already takes it as a **parameter** — it doesn't import it directly. +`createBatch` currently creates it internally via `mockable(createPageMayExitObservable)()`. + +**Decision**: Make `pageMayExitObservable` an explicit parameter of `createBatch` +(consistent with how `encoder` is already injectable). Callers in browser-core +(`startLogsBatch`, `startRumBatch`, `startDebuggerBatch`) will create it and pass it in. + +This avoids moving `pageMayExitObservable` to js-core. + +### 2. How to handle `httpRequest` (uses fetch + sendBeacon) + +Two options: + +- **Option A** _(preferred)_: Move `httpRequest` to js-core as well, making fetch/sendBeacon + strategies injectable via a passed-in `sendStrategy`/`sendOnExitStrategy`. Callers in + browser-core create the strategies and pass them to `createHttpRequest`. + +- **Option B**: Keep `httpRequest` in browser-core and make it an explicit parameter of + `createBatch`. Simpler short-term but breaks the "full stack in js-core" goal. + +### 3. How to handle `timer.ts` (ZoneJS-aware setTimeout) + +`flushController` and `sendWithRetryStrategy` both use `timer.ts`. +`timer.ts` depends on `getZoneJsOriginalValue` and `monitor` (browser-core wrappers). + +**Decision**: Move `timer.ts` and `getZoneJsOriginalValue.ts` to js-core. They already use +`globalObject` from js-core and the ZoneJS concern is valid in any environment. +`monitor` in `timer.ts` wraps callbacks — use injected monitor or js-core/monitor directly. + +### 4. Where does `Observable` live in js-core? + +Include in `@datadog/js-core/util`. It crowds util a bit but avoids a new entry for a single class. + +--- + +## Proposed Commit Sequence + +### Commit 1 — Move pure utilities to js-core/util + +**Files**: `byteUtils.ts`, `responseUtils.ts`, `objectValues` (from `polyfills.ts`), +`functionUtils.ts` (noop + others), `context.ts` (types), `jsonStringify.ts` + +These have **zero browser dependencies** and are used across many modules. +Export via `@datadog/js-core/util` (or a new `@datadog/js-core/serialisation` entry +if preferred to avoid bloating util). + +Update all browser-core imports to re-export from js-core. + +**Why first**: Smallest, most isolated change. No design decisions needed. +Unblocks Encoder and sendWithRetryStrategy moves. + +--- + +### Commit 2 — Move `mockable` to js-core + +**Files**: `mockable.ts` + +Standalone utility, only depends on `__BUILD_ENV__SDK_VERSION__`. +Export from `@datadog/js-core/util` or a new `@datadog/js-core/mockable` entry. + +**Why separate**: Touches many files (anything using mockable) — easier to review in isolation. + +--- + +### Commit 3 — Move `Observable` to js-core + +**Files**: `observable.ts`, `queueMicrotask.ts` + +`Observable` is a fundamental building block used by virtually every module being moved. +`queueMicrotask.ts` needs `monitor` — use js-core/monitor's `createMonitor` directly, +or make the monitor injectable in Observable (simplest: just use `Promise.resolve().then` +as the async scheduling mechanism, removing the monitor dependency in queueMicrotask). + +Export from `@datadog/js-core/util`. + +**Why separate**: High-impact change touching many browser-core imports. Isolated review is valuable. + +--- + +### Commit 4 — Move `Encoder` to js-core/transport + +**Files**: `encoder.ts` (Encoder interface, createIdentityEncoder, EncoderResult) + +Depends only on `byteUtils` (moved in Commit 1). Pure encoding abstraction. +Export from `@datadog/js-core/transport`. + +**Why separate**: Used by batch and by the deflate encoder in browser-worker. +Clean, focused change. + +--- + +### Commit 5 — Move `timer.ts` + `getZoneJsOriginalValue.ts` to js-core/util + +**Files**: `timer.ts`, `getZoneJsOriginalValue.ts` + +Both already use `globalObject` from js-core. Move them alongside it. +Resolve `monitor` dependency: `timer.ts` uses `monitor(callback)` to wrap setTimeout callbacks. +Move `getZoneJsOriginalValue` to js-core/util, use js-core/monitor for wrapping. + +Export `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval`, `TimeoutId` from +`@datadog/js-core/util`. + +**Why separate**: ZoneJS patching is subtle — a dedicated diff is easier to audit. + +--- + +### Commit 6 — Move `sendWithRetryStrategy` to js-core/transport + +**Files**: `sendWithRetryStrategy.ts` + +Depends on: Observable (Commit 3), timer (Commit 5), byteUtils (Commit 1), responseUtils (Commit 1). +Pure retry logic, no browser API calls. Well suited for js-core. + +Export from `@datadog/js-core/transport`. + +--- + +### Commit 7 — Move `flushController` to js-core/transport + +**Files**: `flushController.ts` + +Depends on: Observable (Commit 3), timer (Commit 5). +`pageMayExitObservable` is already injected as a parameter ✓. +Move `RECOMMENDED_REQUEST_BYTES_LIMIT` to js-core/transport (it's a transport constant, +not specific to httpRequest). + +Both `flushController` and `sendWithRetryStrategy` are **internal implementation details** +of `createBatch` — neither needs to be publicly exported from `@datadog/js-core/transport`. +They live as internal modules, only imported by `batch.ts`. + +`FlushEvent`, `FlushReason`, and `UrgentFlushReason` appear in the `Batch` interface +(via `flushObservable` and `prepareUrgentFlushObservable`) so they do need to be exported +as types. `createFlushController` and `FlushController` do not — they stay internal. +`browser-core/src/index.ts` currently re-exports them; after the move it just re-exports +from `@datadog/js-core/transport` instead, with no change for downstream consumers. + +--- + +### Commit 8 — Move `httpRequest` to js-core/transport (Option A) + +**Files**: `httpRequest.ts`, `fetch.ts` (browser-core/browser) + +Make `sendStrategy` and `sendOnExitStrategy` injectable in `createHttpRequest` — the +browser-specific fetch + sendBeacon strategies stay in browser-core and are passed in +by callers. `httpRequest.ts` itself becomes generic logic (retry, observable, queue). + +Alternatively under Option B: skip this commit and inject `httpRequest` as a parameter +to `createBatch` in the next commit, keeping it in browser-core. + +--- + +### Commit 9 — Move `createBatch` to js-core/transport + +**Files**: `batch.ts` + +With all dependencies in js-core, this becomes a straightforward move. +`pageMayExitObservable` becomes an explicit parameter (Decision 1). +Under Option A, `createHttpRequest` is imported from js-core. +Under Option B, `httpRequest` is an injectable parameter. + +Update `browser-core/transport/index.ts` to re-export from js-core for backward compatibility. +Update all callers (`startLogsBatch`, `startRumBatch`, `startDebuggerBatch`) to pass +`pageMayExitObservable` (and optionally `httpRequest` under Option B). + +--- + +## What Stays in browser-core + +| File | Reason | +| ---------------------------------------------------------------- | --------------------------------------------------- | +| `pageMayExitObservable.ts` | Uses DOM events (addEventListener, visibilityState) | +| `fetch.ts` | Uses globalObject.fetch + ZoneJS patching | +| `eventBridge.ts` | Browser-specific | +| `startLogsBatch.ts`, `startRumBatch.ts`, `startDebuggerBatch.ts` | Domain-level wiring | +| `display.ts` | Thin wrapper, can stay as-is | + +--- + +## Notes on Review Strategy + +- Commits 1–5 are the most reviewer-friendly: pure moves with no logic changes. +- Commits 6–7 move logic that already has good test coverage (flushController.spec.ts, + sendWithRetryStrategy.spec.ts) — tests can move with the files. +- The main risk is in Commit 8 (httpRequest refactor) and Commit 9 (batch API change). + These should each be reviewed carefully against the existing `.spec.ts` files. +- After each commit, run `yarn typecheck && yarn test:unit` to verify no regressions. diff --git a/packages/browser-core/src/browser/pageMayExitObservable.ts b/packages/browser-core/src/browser/pageMayExitObservable.ts index b59682b182..d290cee90d 100644 --- a/packages/browser-core/src/browser/pageMayExitObservable.ts +++ b/packages/browser-core/src/browser/pageMayExitObservable.ts @@ -1,20 +1,11 @@ import { globalObject } from '@datadog/js-core/util' +import { PageExitReason } from '@datadog/js-core/transport' +import type { PageMayExitEvent } from '@datadog/js-core/transport' import { Observable } from '../tools/observable' -import { objectValues } from '../tools/utils/polyfills' import { addEventListeners, addEventListener, DOM_EVENT } from './addEventListener' -export const PageExitReason = { - HIDDEN: 'visibility_hidden', - UNLOADING: 'before_unload', - PAGEHIDE: 'page_hide', - FROZEN: 'page_frozen', -} as const - -export type PageExitReason = (typeof PageExitReason)[keyof typeof PageExitReason] - -export interface PageMayExitEvent { - reason: PageExitReason -} +export { PageExitReason, isPageExitReason } from '@datadog/js-core/transport' +export type { PageMayExitEvent } from '@datadog/js-core/transport' export function createPageMayExitObservable(): Observable { return new Observable((observable) => { @@ -54,7 +45,3 @@ export function createPageMayExitObservable(): Observable { } }) } - -export function isPageExitReason(reason: string): reason is PageExitReason { - return objectValues(PageExitReason).includes(reason as PageExitReason) -} diff --git a/packages/browser-core/src/index.ts b/packages/browser-core/src/index.ts index 02befa70e1..b05c1baa75 100644 --- a/packages/browser-core/src/index.ts +++ b/packages/browser-core/src/index.ts @@ -71,7 +71,6 @@ export { bridgeSupports, BridgeCapability, createBatch, - createFlushController, FLUSH_DURATION_LIMIT, } from './transport' export * from './tools/display' diff --git a/packages/browser-core/src/tools/encoder.ts b/packages/browser-core/src/tools/encoder.ts index 4e0ae1997e..9711e47e17 100644 --- a/packages/browser-core/src/tools/encoder.ts +++ b/packages/browser-core/src/tools/encoder.ts @@ -1,103 +1,2 @@ -import type { Uint8ArrayBuffer } from './utils/byteUtils' -import { computeBytesCount } from './utils/byteUtils' - -export interface Encoder { - /** - * Whether this encoder might call the provided callbacks asynchronously - */ - isAsync: boolean - - /** - * Whether some data has been written since the last finish() or finishSync() call - */ - isEmpty: boolean - - /** - * Write a string to be encoded. - * - * This operation can be synchronous or asynchronous depending on the encoder implementation. - * - * If specified, the callback will be invoked when the operation finishes, unless the operation is - * asynchronous and finish() or finishSync() is called in the meantime. - */ - write(data: string, callback?: (additionalEncodedBytesCount: number) => void): void - - /** - * Waits for pending data to be encoded and resets the encoder state. - * - * This operation can be synchronous or asynchronous depending on the encoder implementation. - * - * The callback will be invoked when the operation finishes, unless the operation is asynchronous - * and another call to finish() or finishSync() occurs in the meantime. - */ - finish(callback: (result: EncoderResult) => void): void - - /** - * Resets the encoder state then returns the encoded data and any potential pending data directly, - * discarding all pending write operations and finish() callbacks. - */ - finishSync(): EncoderResult & { pendingData: string } - - /** - * Returns a rough estimation of the bytes count if the data was encoded. - */ - estimateEncodedBytesCount(data: string): number -} - -export interface EncoderResult { - output: Output - outputBytesCount: number - - /** - * An encoding type supported by HTTP Content-Encoding, if applicable. - * See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#directives - */ - encoding?: 'deflate' - - /** - * Total bytes count of the input strings encoded to UTF-8. - */ - rawBytesCount: number -} - -export function createIdentityEncoder(): Encoder { - let output = '' - let outputBytesCount = 0 - - return { - isAsync: false, - - get isEmpty() { - return !output - }, - - write(data, callback) { - const additionalEncodedBytesCount = computeBytesCount(data) - outputBytesCount += additionalEncodedBytesCount - output += data - if (callback) { - callback(additionalEncodedBytesCount) - } - }, - - finish(callback) { - callback(this.finishSync()) - }, - - finishSync() { - const result = { - output, - outputBytesCount, - rawBytesCount: outputBytesCount, - pendingData: '', - } - output = '' - outputBytesCount = 0 - return result - }, - - estimateEncodedBytesCount(data) { - return data.length - }, - } -} +export type { Encoder, EncoderResult } from '@datadog/js-core/transport' +export { createIdentityEncoder } from '@datadog/js-core/transport' diff --git a/packages/browser-core/src/tools/getZoneJsOriginalValue.ts b/packages/browser-core/src/tools/getZoneJsOriginalValue.ts index ca8b4a8e50..9be719c9db 100644 --- a/packages/browser-core/src/tools/getZoneJsOriginalValue.ts +++ b/packages/browser-core/src/tools/getZoneJsOriginalValue.ts @@ -1,38 +1,2 @@ -import { globalObject } from '@datadog/js-core/util' - -export interface BrowserWindowWithZoneJs { - Zone?: { - // All Zone.js versions expose the __symbol__ method, but we observed that some website have a - // 'Zone' global variable unrelated to Zone.js, so let's consider this method optional - // nonetheless. - __symbol__?: (name: string) => string - } -} - -/** - * Gets the original value for a DOM API that was potentially patched by Zone.js. - * - * Zone.js[1] is a library that patches a bunch of JS and DOM APIs. It usually stores the original - * value of the patched functions/constructors/methods in a hidden property prefixed by - * __zone_symbol__. - * - * In multiple occasions, we observed that Zone.js is the culprit of important issues leading to - * browser resource exhaustion (memory leak, high CPU usage). This method is used as a workaround to - * use the original DOM API instead of the one patched by Zone.js. - * - * [1]: https://github.com/angular/angular/tree/main/packages/zone.js - */ -export function getZoneJsOriginalValue( - target: Target, - name: Name -): Target[Name] { - const browserWindow = globalObject as BrowserWindowWithZoneJs - let original: Target[Name] | undefined - if (browserWindow.Zone && typeof browserWindow.Zone.__symbol__ === 'function') { - original = (target as any)[browserWindow.Zone.__symbol__(name)] - } - if (!original) { - original = target[name] - } - return original -} +export type { BrowserWindowWithZoneJs } from '@datadog/js-core/util' +export { getZoneJsOriginalValue } from '@datadog/js-core/util' diff --git a/packages/browser-core/src/tools/mockable.ts b/packages/browser-core/src/tools/mockable.ts index 397ffd82bb..52ab95bff5 100644 --- a/packages/browser-core/src/tools/mockable.ts +++ b/packages/browser-core/src/tools/mockable.ts @@ -1,32 +1 @@ -declare const __BUILD_ENV__SDK_VERSION__: string - -export const mockableReplacements = new Map() - -/** - * Wraps a value to make it mockable in tests. In production builds, this is a no-op - * that returns the value as-is. In test builds, it checks if a mock replacement has - * been registered and returns that instead. - * - * @example - * // In source file: - * import { mockable } from '../tools/mockable' - * export function formatNavigationEntry(): string { - * const navigationEntry = mockable(getNavigationEntry)() - * ... - * } - * - * // In test file: - * import { replaceMockable } from '@datadog/browser-core/test' - * it('...', () => { - * replaceMockable(getNavigationEntry, () => FAKE_NAVIGATION_ENTRY) - * expect(formatNavigationEntry()).toEqual(...) - * }) - */ -export function mockable(value: T): T { - // In test builds, return a wrapper that checks for mocks at call time - if (__BUILD_ENV__SDK_VERSION__ === 'test' && mockableReplacements.has(value)) { - return mockableReplacements.get(value)! as T - } - // In production, return the value as-is - return value -} +export { mockable, mockableReplacements } from '@datadog/js-core/util' diff --git a/packages/browser-core/src/tools/observable.ts b/packages/browser-core/src/tools/observable.ts index deea2a2453..ddb5a0dc6c 100644 --- a/packages/browser-core/src/tools/observable.ts +++ b/packages/browser-core/src/tools/observable.ts @@ -1,112 +1,2 @@ -import { queueMicrotask } from './queueMicrotask' - -export interface Subscription { - unsubscribe: () => void -} - -type Observer = (data: T) => void - -// eslint-disable-next-line no-restricted-syntax -export class Observable { - protected observers: Array> = [] - private onLastUnsubscribe?: () => void - - constructor(private onFirstSubscribe?: (observable: Observable) => (() => void) | void) {} - - subscribe(observer: Observer): Subscription { - this.addObserver(observer) - return { - unsubscribe: () => this.removeObserver(observer), - } - } - - notify(data: T) { - this.observers.forEach((observer) => observer(data)) - } - - protected addObserver(observer: Observer) { - this.observers.push(observer) - if (this.observers.length === 1 && this.onFirstSubscribe) { - this.onLastUnsubscribe = this.onFirstSubscribe(this) || undefined - } - } - - protected removeObserver(observer: Observer) { - this.observers = this.observers.filter((other) => observer !== other) - if (!this.observers.length && this.onLastUnsubscribe) { - this.onLastUnsubscribe() - } - } -} - -export function mergeObservables(...observables: Array>) { - return new Observable((globalObservable) => { - const subscriptions: Subscription[] = observables.map((observable) => - observable.subscribe((data) => globalObservable.notify(data)) - ) - return () => subscriptions.forEach((subscription) => subscription.unsubscribe()) - }) -} - -// eslint-disable-next-line no-restricted-syntax -export class BufferedObservable extends Observable { - private buffer: T[] = [] - private droppedCount = 0 - - constructor( - private maxBufferSize: number, - private onDrop?: (count: number) => void - ) { - super() - } - - notify(data: T) { - this.buffer.push(data) - if (this.buffer.length > this.maxBufferSize) { - this.buffer.shift() - this.droppedCount++ - } - super.notify(data) - } - - subscribe(observer: Observer): Subscription { - let closed = false - - const subscription = { - unsubscribe: () => { - closed = true - this.removeObserver(observer) - }, - } - - queueMicrotask(() => { - for (const data of this.buffer) { - if (closed) { - return - } - observer(data) - } - - if (!closed) { - this.addObserver(observer) - } - }) - - return subscription - } - - /** - * Drop buffered data and don't buffer future data. This is to avoid leaking memory when it's not - * needed anymore. This can be seen as a performance optimization, and things will work probably - * even if this method isn't called, but still useful to clarify our intent and lowering our - * memory impact. - */ - unbuffer() { - queueMicrotask(() => { - if (this.droppedCount > 0 && this.onDrop) { - this.onDrop(this.droppedCount) - } - this.maxBufferSize = this.buffer.length = 0 - }) - } -} +export type { Subscription } from '@datadog/js-core/util' +export { Observable, BufferedObservable, mergeObservables } from '@datadog/js-core/util' diff --git a/packages/browser-core/src/tools/queueMicrotask.spec.ts b/packages/browser-core/src/tools/queueMicrotask.spec.ts deleted file mode 100644 index aa5abc8f3b..0000000000 --- a/packages/browser-core/src/tools/queueMicrotask.spec.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { startMockTelemetry, waitNextMicrotask } from '../../test' -import { queueMicrotask } from './queueMicrotask' - -describe('queueMicrotask', () => { - it('calls the callback in a microtask', async () => { - let called = false - queueMicrotask(() => { - called = true - }) - expect(called).toBe(false) - await waitNextMicrotask() - expect(called).toBe(true) - }) - - it('monitors the callback', async () => { - const telemetry = startMockTelemetry() - queueMicrotask(() => { - throw new Error('test error') - }) - await waitNextMicrotask() - - expect(await telemetry.hasEvents()).toBe(true) - }) -}) diff --git a/packages/browser-core/src/tools/queueMicrotask.ts b/packages/browser-core/src/tools/queueMicrotask.ts deleted file mode 100644 index 5a929ebfcb..0000000000 --- a/packages/browser-core/src/tools/queueMicrotask.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { globalObject } from '@datadog/js-core/util' -import { monitor } from './monitor' - -export function queueMicrotask(callback: () => void) { - // Intentionally avoid .bind(globalObject): in some environments (e.g. Selenium GeckoDriver's - // executeScript), globalThis is not a proper global object, so calling the bound function throws - // 'queueMicrotask called on an object that does not implement interface Window'. Calling it as an - // unbound method is fine, as the proper global object will be used implicitly. - // See https://github.com/mozilla/geckodriver/issues/1798 - const nativeImplementation = globalObject.queueMicrotask - - if (typeof nativeImplementation === 'function') { - nativeImplementation(monitor(callback)) - } else { - // eslint-disable-next-line @typescript-eslint/no-floating-promises -- the callback is monitored, so it'll never throw - Promise.resolve().then(monitor(callback)) - } -} diff --git a/packages/browser-core/src/tools/serialisation/context.ts b/packages/browser-core/src/tools/serialisation/context.ts index 690eb66f12..ee141a54c2 100644 --- a/packages/browser-core/src/tools/serialisation/context.ts +++ b/packages/browser-core/src/tools/serialisation/context.ts @@ -1,11 +1 @@ -export interface Context { - [x: string]: ContextValue -} - -export type ContextValue = string | number | boolean | Context | ContextArray | undefined | null - -/** - * @hidden - */ -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface ContextArray extends Array {} +export type { Context, ContextValue, ContextArray } from '@datadog/js-core/util' diff --git a/packages/browser-core/src/tools/serialisation/jsonStringify.ts b/packages/browser-core/src/tools/serialisation/jsonStringify.ts index 031fe8a7ad..3d150ddf52 100644 --- a/packages/browser-core/src/tools/serialisation/jsonStringify.ts +++ b/packages/browser-core/src/tools/serialisation/jsonStringify.ts @@ -1,53 +1,2 @@ -import { noop } from '../utils/functionUtils' - -/** - * Custom implementation of JSON.stringify that ignores some toJSON methods. We need to do that - * because some sites badly override toJSON on certain objects. Removing all toJSON methods from - * nested values would be too costly, so we just detach them from the root value, and native classes - * used to build JSON values (Array and Object). - * - * Note: this still assumes that JSON.stringify is correct. - */ -export function jsonStringify( - value: unknown, - replacer?: Array, - space?: string | number -): string | undefined { - if (typeof value !== 'object' || value === null) { - return JSON.stringify(value) - } - - // Note: The order matter here. We need to detach toJSON methods on parent classes before their - // subclasses. - const restoreObjectPrototypeToJson = detachToJsonMethod(Object.prototype) - const restoreArrayPrototypeToJson = detachToJsonMethod(Array.prototype) - const restoreValuePrototypeToJson = detachToJsonMethod(Object.getPrototypeOf(value)) - const restoreValueToJson = detachToJsonMethod(value) - - try { - return JSON.stringify(value, replacer, space) - } catch { - return '' - } finally { - restoreObjectPrototypeToJson() - restoreArrayPrototypeToJson() - restoreValuePrototypeToJson() - restoreValueToJson() - } -} - -export interface ObjectWithToJsonMethod { - toJSON?: () => unknown -} - -export function detachToJsonMethod(value: object) { - const object = value as ObjectWithToJsonMethod - const objectToJson = object.toJSON - if (objectToJson) { - delete object.toJSON - return () => { - object.toJSON = objectToJson - } - } - return noop -} +export { jsonStringify, detachToJsonMethod } from '@datadog/js-core/util' +export type { ObjectWithToJsonMethod } from '@datadog/js-core/util' diff --git a/packages/browser-core/src/tools/timer.ts b/packages/browser-core/src/tools/timer.ts index 13efb473ce..3890d1e815 100644 --- a/packages/browser-core/src/tools/timer.ts +++ b/packages/browser-core/src/tools/timer.ts @@ -1,9 +1,8 @@ -import type { GlobalObject } from '@datadog/js-core/util' -import { globalObject } from '@datadog/js-core/util' -import { getZoneJsOriginalValue } from './getZoneJsOriginalValue' +import type { TimeoutId } from '@datadog/js-core/util' +import { getZoneJsOriginalValue, globalObject } from '@datadog/js-core/util' import { monitor } from './monitor' -export type TimeoutId = ReturnType +export type { TimeoutId } export function setTimeout(callback: () => void, delay?: number): TimeoutId { return getZoneJsOriginalValue(globalObject, 'setTimeout')(monitor(callback), delay) diff --git a/packages/browser-core/src/tools/utils/byteUtils.ts b/packages/browser-core/src/tools/utils/byteUtils.ts index 3f44f21eed..c12fe27467 100644 --- a/packages/browser-core/src/tools/utils/byteUtils.ts +++ b/packages/browser-core/src/tools/utils/byteUtils.ts @@ -1,36 +1,2 @@ -export const ONE_KIBI_BYTE = 1024 -export const ONE_MEBI_BYTE = 1024 * ONE_KIBI_BYTE - -// eslint-disable-next-line no-control-regex -const HAS_MULTI_BYTES_CHARACTERS = /[^\u0000-\u007F]/ - -export interface Uint8ArrayBuffer extends Uint8Array { - readonly buffer: ArrayBuffer - - subarray(begin?: number, end?: number): Uint8ArrayBuffer -} - -export function computeBytesCount(candidate: string): number { - // Accurate bytes count computations can degrade performances when there is a lot of events to process - if (!HAS_MULTI_BYTES_CHARACTERS.test(candidate)) { - return candidate.length - } - - return new TextEncoder().encode(candidate).length -} - -export function concatBuffers(buffers: Uint8ArrayBuffer[]): Uint8ArrayBuffer { - // Optimization: if there is a single buffer, no need to copy it - if (buffers.length === 1) { - return buffers[0] - } - - const length = buffers.reduce((total, buffer) => total + buffer.length, 0) - const result: Uint8ArrayBuffer = new Uint8Array(length) - let offset = 0 - for (const buffer of buffers) { - result.set(buffer, offset) - offset += buffer.length - } - return result -} +export { ONE_KIBI_BYTE, ONE_MEBI_BYTE, computeBytesCount, concatBuffers } from '@datadog/js-core/util' +export type { Uint8ArrayBuffer } from '@datadog/js-core/util' diff --git a/packages/browser-core/src/tools/utils/polyfills.ts b/packages/browser-core/src/tools/utils/polyfills.ts index 907023fd8d..f8408a2d99 100644 --- a/packages/browser-core/src/tools/utils/polyfills.ts +++ b/packages/browser-core/src/tools/utils/polyfills.ts @@ -1,3 +1,5 @@ +export { objectValues } from '@datadog/js-core/util' + export function findLast( array: readonly T[], predicate: (item: T, index: number, array: readonly T[]) => item is S @@ -14,10 +16,6 @@ export function findLast( // Keep the following wrapper functions as it can be mangled and will result in smaller bundle size that using // the native Object.values and Object.entries directly -export function objectValues(object: { [key: string]: T }) { - return Object.values(object) -} - export function objectEntries(object: { [key: string]: T }): Array<[string, T]> { return Object.entries(object) } diff --git a/packages/browser-core/src/tools/utils/responseUtils.ts b/packages/browser-core/src/tools/utils/responseUtils.ts index fb98fb503e..2a305d909a 100644 --- a/packages/browser-core/src/tools/utils/responseUtils.ts +++ b/packages/browser-core/src/tools/utils/responseUtils.ts @@ -1,6 +1,4 @@ -export function isServerError(status: number) { - return status >= 500 -} +export { isServerError } from '@datadog/js-core/util' export function tryToClone(response: Response): Response | undefined { try { diff --git a/packages/browser-core/src/transport/batch.ts b/packages/browser-core/src/transport/batch.ts index 99a79cbdac..9409e92b68 100644 --- a/packages/browser-core/src/transport/batch.ts +++ b/packages/browser-core/src/transport/batch.ts @@ -1,33 +1,30 @@ -import type { EndpointBuilder } from '@datadog/js-core/transport' +import { createBatch as jsCreateBatch, MESSAGE_BYTES_LIMIT } from '@datadog/js-core/transport' +import type { EndpointBuilder, Batch } from '@datadog/js-core/transport' +import { createPageMayExitObservable } from '../browser/pageMayExitObservable' import { DOCS_TROUBLESHOOTING, MORE_DETAILS, display } from '../tools/display' -import type { Context } from '../tools/serialisation/context' -import { objectValues } from '../tools/utils/polyfills' -import { isPageExitReason, createPageMayExitObservable } from '../browser/pageMayExitObservable' -import { jsonStringify } from '../tools/serialisation/jsonStringify' -import { createIdentityEncoder } from '../tools/encoder' -import type { Encoder, EncoderResult } from '../tools/encoder' -import { computeBytesCount, ONE_KIBI_BYTE } from '../tools/utils/byteUtils' +import type { Encoder } from '../tools/encoder' import { mockable } from '../tools/mockable' -import type { Observable } from '../tools/observable' import { createHttpRequest } from './httpRequest' -import type { Payload } from './httpRequest' -import { createFlushController } from './flushController' -import type { FlushEvent, FlushReason, UrgentFlushReason } from './flushController' - -export const MESSAGE_BYTES_LIMIT = 256 * ONE_KIBI_BYTE - -export interface Batch { - isEmpty: boolean - add: (message: Context) => void - upsert: (message: Context, key: string) => void - forceFlush: (reason: FlushReason) => void - prepareUrgentFlushObservable: Observable - flushObservable: Observable - stop: () => void -} +export type { Batch } +export { MESSAGE_BYTES_LIMIT } + +/** + * Creates a batch wired to the browser's HTTP transport and page-exit detection. + * + * This is a thin browser-core wrapper around the generic `createBatch` from js-core. + * It injects: + * - an `HttpRequest` built from `endpoints` using the browser fetch / sendBeacon strategies + * - a `pageMayExitObservable` backed by DOM visibility and beforeunload events + * - `display.warn` for oversized-message warnings (with the troubleshooting docs URL) + * + * @param options - See parameter descriptions below. + * @param options.endpoints - Intake endpoint builders to send to. + * @param options.reportError - Called when the send queue overflows. + * @param options.encoder - Optional encoder; defaults to the identity encoder. + */ export function createBatch({ - encoder = createIdentityEncoder(), + encoder, endpoints, reportError, }: { @@ -37,116 +34,12 @@ export function createBatch({ }): Batch { const request = mockable(createHttpRequest)(endpoints, reportError) const pageMayExitObservable = mockable(createPageMayExitObservable)() - const flushController = mockable(createFlushController)({ pageMayExitObservable }) - let upsertBuffer: { [key: string]: string } = {} - const flushSubscription = flushController.flushObservable.subscribe((event) => flush(event)) - - function push(serializedMessage: string, estimatedMessageBytesCount: number, key?: string) { - if (key !== undefined) { - let bytesDiff: number - if (upsertBuffer[key] !== undefined) { - bytesDiff = estimatedMessageBytesCount - encoder.estimateEncodedBytesCount(upsertBuffer[key]) - } else { - flushController.notifyBeforeAddMessage(estimatedMessageBytesCount) - bytesDiff = 0 - } - upsertBuffer[key] = serializedMessage - flushController.notifyAfterAddMessage(bytesDiff) - } else { - flushController.notifyBeforeAddMessage(estimatedMessageBytesCount) - encoder.write(encoder.isEmpty ? serializedMessage : `\n${serializedMessage}`, (realMessageBytesCount) => { - flushController.notifyAfterAddMessage(realMessageBytesCount - estimatedMessageBytesCount) - }) - } - } - - function addOrUpdate(message: Context, key?: string) { - const serializedMessage = jsonStringify(message)! - - const estimatedMessageBytesCount = encoder.estimateEncodedBytesCount(serializedMessage) - - if (estimatedMessageBytesCount >= MESSAGE_BYTES_LIMIT) { - display.warn( - `Discarded a message whose size was bigger than the maximum allowed size ${MESSAGE_BYTES_LIMIT / ONE_KIBI_BYTE}KiB. ${MORE_DETAILS} ${DOCS_TROUBLESHOOTING}/#technical-limitations` - ) - return - } - - push(serializedMessage, estimatedMessageBytesCount, key) - } - - function flush(event: FlushEvent) { - const upsertMessages = objectValues(upsertBuffer).join('\n') - upsertBuffer = {} - - const pageMightExit = isPageExitReason(event.reason) - const send = pageMightExit ? request.sendOnExit : request.send - - if ( - pageMightExit && - // Note: checking that the encoder is async is not strictly needed, but it's an optimization: - // if the encoder is async we need to send two requests in some cases (one for encoded data - // and the other for non-encoded data). But if it's not async, we don't have to worry about - // it and always send a single request. - encoder.isAsync - ) { - const encoderResult = encoder.finishSync() - - // Send encoded messages - if (encoderResult.outputBytesCount) { - send(formatPayloadFromEncoder(encoderResult)) - } - - // Send messages that are not yet encoded at this point - const pendingMessages = [encoderResult.pendingData, upsertMessages].filter(Boolean).join('\n') - if (pendingMessages) { - send({ - data: pendingMessages, - bytesCount: computeBytesCount(pendingMessages), - }) - } - } else { - if (upsertMessages) { - encoder.write(encoder.isEmpty ? upsertMessages : `\n${upsertMessages}`) - } - encoder.finish((encoderResult) => { - send(formatPayloadFromEncoder(encoderResult)) - }) - } - } - - return { - get isEmpty() { - return flushController.messagesCount === 0 - }, - add: addOrUpdate, - upsert: addOrUpdate, - prepareUrgentFlushObservable: flushController.prepareUrgentFlushObservable, - forceFlush: flushController.forceFlush, - flushObservable: flushController.flushObservable, - stop: flushSubscription.unsubscribe, - } -} - -function formatPayloadFromEncoder(encoderResult: EncoderResult): Payload { - let data: string | Blob - if (typeof encoderResult.output === 'string') { - data = encoderResult.output - } else { - data = new Blob([encoderResult.output], { - // This will set the 'Content-Type: text/plain' header. Reasoning: - // * The intake rejects the request if there is no content type. - // * The browser will issue CORS preflight requests if we set it to 'application/json', which - // could induce higher intake load (and maybe has other impacts). - // * Also it's not quite JSON, since we are concatenating multiple JSON objects separated by - // new lines. - type: 'text/plain', - }) - } - return { - data, - bytesCount: encoderResult.outputBytesCount, - encoding: encoderResult.encoding, - } + return jsCreateBatch({ + request, + pageMayExitObservable, + encoder, + reportError, + warn: (message) => display.warn(`${message} ${MORE_DETAILS} ${DOCS_TROUBLESHOOTING}/#technical-limitations`), + }) } diff --git a/packages/browser-core/src/transport/flushController.ts b/packages/browser-core/src/transport/flushController.ts index a408366ab2..1834355c3a 100644 --- a/packages/browser-core/src/transport/flushController.ts +++ b/packages/browser-core/src/transport/flushController.ts @@ -1,142 +1,2 @@ -import { ONE_SECOND } from '@datadog/js-core/time' -import type { Duration } from '@datadog/js-core/time' -import { isWorkerEnvironment } from '@datadog/js-core/util' -import type { PageMayExitEvent, PageExitReason } from '../browser/pageMayExitObservable' -import { Observable } from '../tools/observable' -import type { TimeoutId } from '../tools/timer' -import { clearTimeout, setTimeout } from '../tools/timer' -import { RECOMMENDED_REQUEST_BYTES_LIMIT } from './httpRequest' - -export type UrgentFlushReason = PageExitReason -export type FlushReason = UrgentFlushReason | 'duration_limit' | 'bytes_limit' | 'messages_limit' | 'session_expire' - -/** - * flush automatically, aim to be lower than ALB connection timeout - * to maximize connection reuse. - */ -export const FLUSH_DURATION_LIMIT = (30 * ONE_SECOND) as Duration - -/** - * When using the SDK in a Worker Environment, we limit the batch size to 1 to ensure it can be sent - * in a single event. - */ -export const MESSAGES_LIMIT = isWorkerEnvironment ? 1 : 50 - -export type FlushController = ReturnType -export interface FlushEvent { - reason: FlushReason - bytesCount: number - messagesCount: number -} - -interface FlushControllerOptions { - pageMayExitObservable: Observable -} - -/** - * Returns a "flush controller", responsible of notifying when flushing a pool of pending data needs - * to happen. The implementation is designed to support both synchronous and asynchronous usages, - * but relies on invariants described in each method documentation to keep a coherent state. - */ -export function createFlushController({ pageMayExitObservable }: FlushControllerOptions) { - let forcedFlushReason: FlushReason | undefined - const prepareUrgentFlushObservable = new Observable() - const pageMayExitSubscription = pageMayExitObservable.subscribe((event) => { - forcedFlushReason = event.reason - try { - prepareUrgentFlushObservable.notify(event.reason) - } finally { - forcedFlushReason = undefined - } - flush(event.reason) - }) - - const flushObservable = new Observable(() => () => { - pageMayExitSubscription.unsubscribe() - }) - - let currentBytesCount = 0 - let currentMessagesCount = 0 - - function flush(flushReason: FlushReason) { - if (currentMessagesCount === 0) { - return - } - - const messagesCount = currentMessagesCount - const bytesCount = currentBytesCount - - currentMessagesCount = 0 - currentBytesCount = 0 - cancelDurationLimitTimeout() - - flushObservable.notify({ - reason: flushReason, - messagesCount, - bytesCount, - }) - } - - let durationLimitTimeoutId: TimeoutId | undefined - function scheduleDurationLimitTimeout() { - if (durationLimitTimeoutId === undefined) { - durationLimitTimeoutId = setTimeout(() => { - flush('duration_limit') - }, FLUSH_DURATION_LIMIT) - } - } - - function cancelDurationLimitTimeout() { - clearTimeout(durationLimitTimeoutId) - durationLimitTimeoutId = undefined - } - - return { - flushObservable, - prepareUrgentFlushObservable, - forceFlush: flush, - get messagesCount() { - return currentMessagesCount - }, - - /** - * Notifies that a message will be added to a pool of pending messages waiting to be flushed. - * - * This function needs to be called synchronously, right before adding the message, so no flush - * event can happen after `notifyBeforeAddMessage` and before adding the message. - * - * @param estimatedMessageBytesCount - an estimation of the message bytes count once it is - * actually added. - */ - notifyBeforeAddMessage(estimatedMessageBytesCount: number) { - if (currentBytesCount + estimatedMessageBytesCount >= RECOMMENDED_REQUEST_BYTES_LIMIT) { - flush(forcedFlushReason ?? 'bytes_limit') - } - // Consider the message to be added now rather than in `notifyAfterAddMessage`, because if no - // message was added yet and `notifyAfterAddMessage` is called asynchronously, we still want - // to notify when a flush is needed (for example on page exit). - currentMessagesCount += 1 - currentBytesCount += estimatedMessageBytesCount - scheduleDurationLimitTimeout() - }, - - /** - * Notifies that a message *was* added to a pool of pending messages waiting to be flushed. - * - * This function can be called asynchronously after the message was added, but in this case it - * should not be called if a flush event occurred in between. - * - * @param messageBytesCountDiff - the difference between the estimated message bytes count and - * its actual bytes count once added to the pool. - */ - notifyAfterAddMessage(messageBytesCountDiff = 0) { - currentBytesCount += messageBytesCountDiff - - if (currentMessagesCount >= MESSAGES_LIMIT) { - flush(forcedFlushReason ?? 'messages_limit') - } else if (currentBytesCount >= RECOMMENDED_REQUEST_BYTES_LIMIT) { - flush(forcedFlushReason ?? 'bytes_limit') - } - }, - } -} +export { PageExitReason, FLUSH_DURATION_LIMIT } from '@datadog/js-core/transport' +export type { PageMayExitEvent, UrgentFlushReason, FlushReason, FlushEvent } from '@datadog/js-core/transport' diff --git a/packages/browser-core/src/transport/httpRequest.spec.ts b/packages/browser-core/src/transport/httpRequest.spec.ts index 0f1efd1bb2..38c3f90aeb 100644 --- a/packages/browser-core/src/transport/httpRequest.spec.ts +++ b/packages/browser-core/src/transport/httpRequest.spec.ts @@ -1,5 +1,5 @@ -import type { EndpointBuilder } from '@datadog/js-core/transport' -import { createEndpointBuilder } from '@datadog/js-core/transport' +import type { EndpointBuilder, HttpRequestEvent } from '@datadog/js-core/transport' +import { createEndpointBuilder, RECOMMENDED_REQUEST_BYTES_LIMIT } from '@datadog/js-core/transport' import type { Request } from '../../test' import { collectAsyncCalls, @@ -11,8 +11,8 @@ import { wait, } from '../../test' import { noop } from '../tools/utils/functionUtils' -import type { HttpRequest, HttpRequestEvent } from './httpRequest' -import { createHttpRequest, fetchStrategy, RECOMMENDED_REQUEST_BYTES_LIMIT } from './httpRequest' +import type { HttpRequest } from './httpRequest' +import { createHttpRequest, fetchStrategy } from './httpRequest' describe('httpRequest', () => { const ENDPOINT_URL = 'http://my.website' diff --git a/packages/browser-core/src/transport/httpRequest.ts b/packages/browser-core/src/transport/httpRequest.ts index d7252b4091..26fcd879be 100644 --- a/packages/browser-core/src/transport/httpRequest.ts +++ b/packages/browser-core/src/transport/httpRequest.ts @@ -1,103 +1,48 @@ -import type { EndpointBuilder, TransportRetryInfo } from '@datadog/js-core/transport' -import type { Context } from '../tools/serialisation/context' +import { + createHttpRequest as jsCoreCreateHttpRequest, + RECOMMENDED_REQUEST_BYTES_LIMIT, +} from '@datadog/js-core/transport' +import type { EndpointBuilder, Payload, HttpResponse } from '@datadog/js-core/transport' import { fetch } from '../browser/fetch' import { monitor, monitorError } from '../tools/monitor' -import { Observable } from '../tools/observable' -import { ONE_KIBI_BYTE } from '../tools/utils/byteUtils' -import { newRetryState, sendWithRetryStrategy } from './sendWithRetryStrategy' -/** - * beacon payload max queue size implementation is 64kb - * ensure that we leave room for logs, rum and potential other users - */ -export const RECOMMENDED_REQUEST_BYTES_LIMIT = 16 * ONE_KIBI_BYTE +export type { HttpRequest } from '@datadog/js-core/transport' +export { RECOMMENDED_REQUEST_BYTES_LIMIT } /** - * Use POST request without content type to: - * - avoid CORS preflight requests - * - allow usage of sendBeacon + * Creates an {@link HttpRequest} wired to the browser's fetch and sendBeacon APIs. + * + * This is the browser-core entry point for building intake HTTP requests. It injects + * {@link fetchStrategy} and {@link sendBeaconStrategy} into the generic js-core + * implementation so it can be tested without browser globals. * - * multiple elements are sent separated by \n in order - * to be parsed correctly without content type header + * @param endpointBuilders - Intake endpoints to target. + * @param reportError - Called when the send queue overflows. + * @param bytesLimit - Beacon size limit; defaults to {@link RECOMMENDED_REQUEST_BYTES_LIMIT}. */ - -export interface HttpRequest { - observable: Observable> - send(this: void, payload: Body): void - sendOnExit(this: void, payload: Body): void -} - -export interface HttpResponse extends Context { - status: number - type?: ResponseType -} - -export interface BandwidthStats { - ongoingByteCount: number - ongoingRequestCount: number -} - -export type HttpRequestEvent = - | { - // A request to send the given payload failed. (We may retry.) - type: 'failure' - bandwidth: BandwidthStats - payload: Body - } - | { - // The given payload was discarded because the request queue is full. - type: 'queue-full' - bandwidth: BandwidthStats - payload: Body - } - | { - // A request to send the given payload succeeded. - type: 'success' - bandwidth: BandwidthStats - payload: Body - } - -export interface Payload { - data: string | FormData | Blob - bytesCount: number - retry?: TransportRetryInfo - encoding?: 'deflate' -} - export function createHttpRequest( endpointBuilders: EndpointBuilder[], reportError: (message: string) => void, bytesLimit: number = RECOMMENDED_REQUEST_BYTES_LIMIT -): HttpRequest { - const observable = new Observable>() - const retryState = newRetryState() +) { + return jsCoreCreateHttpRequest( + endpointBuilders, + reportError, + (endpointBuilder, payload, onResponse) => fetchStrategy(endpointBuilder, payload, onResponse), + (endpointBuilder, payload) => sendBeaconStrategy(endpointBuilder, bytesLimit, payload) + ) +} - return { - observable, - send: (payload: Body) => { - for (const endpointBuilder of endpointBuilders) { - sendWithRetryStrategy( - payload, - retryState, - (payload, onResponse) => { - fetchStrategy(endpointBuilder, payload, onResponse) - }, - endpointBuilder.trackType, - reportError, - observable - ) - } - }, - /** - * Since fetch keepalive behaves like regular fetch on Firefox, - * keep using sendBeaconStrategy on exit - */ - sendOnExit: (payload: Body) => { - for (const endpointBuilder of endpointBuilders) { - sendBeaconStrategy(endpointBuilder, bytesLimit, payload) - } - }, - } +export function fetchStrategy( + endpointBuilder: EndpointBuilder, + payload: Payload, + onResponse?: (r: HttpResponse) => void +) { + const fetchUrl = endpointBuilder.build('fetch', payload) + + fetch(fetchUrl, { method: 'POST', body: payload.data, mode: 'cors' }) + .then(monitor((response: Response) => onResponse?.({ status: response.status, type: response.type }))) + .catch(monitor(() => onResponse?.({ status: 0 }))) } function sendBeaconStrategy(endpointBuilder: EndpointBuilder, bytesLimit: number, payload: Payload) { @@ -106,7 +51,6 @@ function sendBeaconStrategy(endpointBuilder: EndpointBuilder, bytesLimit: number try { const beaconUrl = endpointBuilder.build('beacon', payload) const isQueued = navigator.sendBeacon(beaconUrl, payload.data) - if (isQueued) { return } @@ -126,15 +70,3 @@ function reportBeaconError(e: unknown) { monitorError(e) } } - -export function fetchStrategy( - endpointBuilder: EndpointBuilder, - payload: Payload, - onResponse?: (r: HttpResponse) => void -) { - const fetchUrl = endpointBuilder.build('fetch', payload) - - fetch(fetchUrl, { method: 'POST', body: payload.data, mode: 'cors' }) - .then(monitor((response: Response) => onResponse?.({ status: response.status, type: response.type }))) - .catch(monitor(() => onResponse?.({ status: 0 }))) -} diff --git a/packages/browser-core/src/transport/index.ts b/packages/browser-core/src/transport/index.ts index 5cdccd6b6f..899575bc12 100644 --- a/packages/browser-core/src/transport/index.ts +++ b/packages/browser-core/src/transport/index.ts @@ -1,8 +1,9 @@ -export type { BandwidthStats, HttpRequest, HttpRequestEvent, Payload } from './httpRequest' +export type { BandwidthStats, HttpRequestEvent, Payload } from '@datadog/js-core/transport' +export type { HttpRequest } from './httpRequest' export { createHttpRequest } from './httpRequest' export type { BrowserWindowWithEventBridge, DatadogEventBridge } from './eventBridge' export { canUseEventBridge, bridgeSupports, getEventBridge, BridgeCapability } from './eventBridge' export type { Batch } from './batch' export { createBatch } from './batch' -export type { FlushController, FlushEvent, FlushReason, UrgentFlushReason } from './flushController' -export { createFlushController, FLUSH_DURATION_LIMIT } from './flushController' +export type { FlushEvent, FlushReason, UrgentFlushReason } from './flushController' +export { FLUSH_DURATION_LIMIT } from './flushController' diff --git a/packages/browser-core/test/index.ts b/packages/browser-core/test/index.ts index 7d72353f84..399b6edba9 100644 --- a/packages/browser-core/test/index.ts +++ b/packages/browser-core/test/index.ts @@ -14,7 +14,6 @@ export * from './emulate/mockSyntheticsWorkerValues' export * from './emulate/mockVisibilityState' export * from './emulate/mockNavigator' export * from './emulate/mockEventBridge' -export * from './emulate/mockFlushController' export * from './emulate/mockFetch' export * from './emulate/mockXhr' export * from './emulate/mockEventTarget' diff --git a/packages/js-core/api/transport.api.md b/packages/js-core/api/transport.api.md index c474c34374..ee304cfa56 100644 --- a/packages/js-core/api/transport.api.md +++ b/packages/js-core/api/transport.api.md @@ -4,6 +4,23 @@ ```ts +// @public +export interface BandwidthStats { + ongoingByteCount: number; + ongoingRequestCount: number; +} + +// @public +export interface Batch { + add: (message: Context) => void; + flushObservable: Observable; + forceFlush: (reason: FlushReason) => void; + isEmpty: boolean; + prepareUrgentFlushObservable: Observable; + stop: () => void; + upsert: (message: Context, key: string) => void; +} + // @public export function buildEndpointUrl(input: BuildEndpointUrlOptions): string; @@ -16,12 +33,48 @@ export interface BuildEndpointUrlOptions { subdomain?: string; } +// @public +export function createBatch(input: { + request: HttpRequest; + pageMayExitObservable: Observable; + endpoints?: EndpointBuilder[]; + reportError: (message: string) => void; + warn: (message: string) => void; + encoder?: Encoder; +}): Batch; + // @public export function createEndpointBuilder(configuration: EndpointBuilderConfiguration, trackType: TrackType, extraParameters?: string[]): EndpointBuilder; +// @public +export function createHttpRequest(endpointBuilders: EndpointBuilder[], reportError: (message: string) => void, sendStrategy: SendStrategy, sendOnExitStrategy: SendOnExitStrategy): HttpRequest; + +// @public +export function createIdentityEncoder(): Encoder; + // @public export function createReplicaEndpointBuilder(input: ConfigurationWithReplica, trackType: TrackType): EndpointBuilder | undefined; +// @public +export interface Encoder { + estimateEncodedBytesCount(data: string): number; + finish(callback: (result: EncoderResult) => void): void; + finishSync(): EncoderResult & { + pendingData: string; + }; + isAsync: boolean; + isEmpty: boolean; + write(data: string, callback?: (additionalEncodedBytesCount: number) => void): void; +} + +// @public +export interface EncoderResult { + encoding?: 'deflate'; + output: Output; + outputBytesCount: number; + rawBytesCount: number; +} + // @public export interface EndpointBuilder { build(api: TransportApiType, payload: EndpointPayload): string; @@ -34,6 +87,49 @@ export interface EndpointPayload { retry?: TransportRetryInfo; } +// @public +export const FLUSH_DURATION_LIMIT: Duration; + +// @public +export interface FlushEvent { + bytesCount: number; + messagesCount: number; + reason: FlushReason; +} + +// @public +export type FlushReason = UrgentFlushReason | 'duration_limit' | 'bytes_limit' | 'messages_limit' | 'session_expire'; + +// @public +export interface HttpRequest { + observable: Observable>; + send(this: void, payload: Body): void; + sendOnExit(this: void, payload: Body): void; +} + +// @public +export type HttpRequestEvent = { + type: 'failure'; + bandwidth: BandwidthStats; + payload: Body; +} | { + type: 'queue-full'; + bandwidth: BandwidthStats; + payload: Body; +} | { + type: 'success'; + bandwidth: BandwidthStats; + payload: Body; +}; + +// @public +export interface HttpResponse extends Context { + // (undocumented) + status: number; + // (undocumented) + type?: ResponseType; +} + // @public export const INTAKE_SITE_EU1: Site; @@ -55,6 +151,41 @@ export const INTAKE_URL_PARAMETERS: string[]; // @public export function isIntakeUrl(url: string): boolean; +// @public +export function isPageExitReason(reason: string): reason is PageExitReason; + +// @public +export const MESSAGE_BYTES_LIMIT: number; + +// @public +export const PageExitReason: { + readonly HIDDEN: "visibility_hidden"; + readonly UNLOADING: "before_unload"; + readonly PAGEHIDE: "page_hide"; + readonly FROZEN: "page_frozen"; +}; + +// @public (undocumented) +export type PageExitReason = (typeof PageExitReason)[keyof typeof PageExitReason]; + +// @public +export interface PageMayExitEvent { + // (undocumented) + reason: PageExitReason; +} + +// @public +export interface Payload { + // (undocumented) + bytesCount: number; + // (undocumented) + data: string | FormData | Blob; + // (undocumented) + encoding?: 'deflate'; + // (undocumented) + retry?: TransportRetryInfo; +} + // @public export type ProxyFn = (options: { path: string; @@ -62,6 +193,15 @@ export type ProxyFn = (options: { subdomain?: string; }) => string; +// @public +export const RECOMMENDED_REQUEST_BYTES_LIMIT: number; + +// @public +export type SendOnExitStrategy = (endpointBuilder: EndpointBuilder, payload: Body) => void; + +// @public +export type SendStrategy = (endpointBuilder: EndpointBuilder, payload: Body, onResponse: (response: HttpResponse) => void) => void; + // @public export type Site = 'datadoghq.com' | 'us3.datadoghq.com' | 'us5.datadoghq.com' | 'datadoghq.eu' | 'ddog-gov.com' | 'us2.ddog-gov.com' | 'ap1.datadoghq.com' | 'ap2.datadoghq.com' | (string & {}); @@ -80,6 +220,9 @@ export interface TransportRetryInfo { // @public export type TransportSource = 'browser' | 'flutter' | 'unity' | 'dd_debugger'; +// @public +export type UrgentFlushReason = PageExitReason; + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/js-core/api/util.api.md b/packages/js-core/api/util.api.md index 32e2528ad6..a0f30bb508 100644 --- a/packages/js-core/api/util.api.md +++ b/packages/js-core/api/util.api.md @@ -4,9 +4,35 @@ ```ts +// @public +export interface BrowserWindowWithZoneJs { + // (undocumented) + Zone?: { + __symbol__?: (name: string) => string; + }; +} + +// @public +export class BufferedObservable extends Observable { + constructor(maxBufferSize: number, onDrop?: ((count: number) => void) | undefined); + // (undocumented) + notify(data: T): void; + // (undocumented) + subscribe(observer: Observer): Subscription; + unbuffer(): void; +} + // @public export function buildUrl(url: string, base?: string): URL; +// @public +function clearInterval_2(timeoutId: TimeoutId | undefined): void; +export { clearInterval_2 as clearInterval } + +// @public +function clearTimeout_2(timeoutId: TimeoutId | undefined): void; +export { clearTimeout_2 as clearTimeout } + // @public export function combine(a: A, b: B): Combined; @@ -28,6 +54,12 @@ export function combine(a: A, b: B, c: C, d: D, e: E, f: F, // @public (undocumented) export function combine(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H): Combined, C>, D>, E>, F>, G>, H>; +// @public +export function computeBytesCount(candidate: string): number; + +// @public +export function concatBuffers(buffers: Uint8ArrayBuffer[]): Uint8ArrayBuffer; + // @public export const ConsoleApiName: { readonly log: "log"; @@ -40,6 +72,19 @@ export const ConsoleApiName: { // @public export type ConsoleApiName = (typeof ConsoleApiName)[keyof typeof ConsoleApiName]; +// @public +export interface Context { + // (undocumented) + [x: string]: ContextValue; +} + +// @public +export interface ContextArray extends Array { +} + +// @public +export type ContextValue = string | number | boolean | Context | ContextArray | undefined | null; + // @public type CookieChangeEvent_2 = Event & { changed: CookieChangeItem[]; @@ -105,6 +150,9 @@ export function createDisplay(prefix: string): Display; // @public export function deepClone(value: T): T; +// @public +export function detachToJsonMethod(value: object): () => void; + // @public export interface Display { // (undocumented) @@ -131,6 +179,9 @@ export function getPristineWindow(): Pick; // @public export function getType(value: unknown): "string" | "number" | "bigint" | "boolean" | "symbol" | "undefined" | "object" | "function" | "null" | "array"; +// @public +export function getZoneJsOriginalValue(target: Target, name: Name): Target[Name]; + // @public export const globalConsole: Console; @@ -149,15 +200,30 @@ export const globalObject: GlobalObject; // @public export function isIndexableObject(value: unknown): value is Record; +// @public +export function isServerError(status: number): boolean; + // @public export function isValidUrl(url: string): boolean; // @public export const isWorkerEnvironment: boolean; +// @public +export function jsonStringify(value: unknown, replacer?: Array, space?: string | number): string | undefined; + // @public export function mergeInto(destination: D, source: S): Merged; +// @public +export function mergeObservables(...observables: Array>): Observable; + +// @public +export function mockable(value: T): T; + +// @public +export const mockableReplacements: Map; + // @public interface Navigator_2 { connection?: NetworkInformation; @@ -181,6 +247,36 @@ export type NetworkInterface = 'bluetooth' | 'cellular' | 'ethernet' | 'none' | // @public export function normalizeUrl(url: string): string; +// @public +export function objectValues(object: { + [key: string]: T; +}): T[]; + +// @public +export interface ObjectWithToJsonMethod { + // (undocumented) + toJSON?: () => unknown; +} + +// @public +export class Observable { + constructor(onFirstSubscribe?: ((observable: Observable) => (() => void) | void) | undefined); + // (undocumented) + protected addObserver(observer: Observer): void; + notify(data: T): void; + // (undocumented) + protected observers: Array>; + // (undocumented) + protected removeObserver(observer: Observer): void; + subscribe(observer: Observer): Subscription; +} + +// @public +export const ONE_KIBI_BYTE = 1024; + +// @public +export const ONE_MEBI_BYTE: number; + // @public export const originalConsoleMethods: Display; @@ -250,6 +346,31 @@ export interface SampleBufferFullEvent extends Event { // @public export function setDebugMode(newDebugMode: boolean): void; +// @public +function setInterval_2(callback: () => void, delay?: number): TimeoutId; +export { setInterval_2 as setInterval } + +// @public +function setTimeout_2(callback: () => void, delay?: number): TimeoutId; +export { setTimeout_2 as setTimeout } + +// @public +export interface Subscription { + // (undocumented) + unsubscribe: () => void; +} + +// @public +export type TimeoutId = ReturnType; + +// @public +export interface Uint8ArrayBuffer extends Uint8Array { + // (undocumented) + readonly buffer: ArrayBuffer; + // (undocumented) + subarray(begin?: number, end?: number): Uint8ArrayBuffer; +} + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/js-core/src/entries/transport.ts b/packages/js-core/src/entries/transport.ts index c5454735a3..196868a2f7 100644 --- a/packages/js-core/src/entries/transport.ts +++ b/packages/js-core/src/entries/transport.ts @@ -9,6 +9,17 @@ export type { EndpointPayload, } from '../transport/endpointBuilder' export { createEndpointBuilder, createReplicaEndpointBuilder, buildEndpointUrl } from '../transport/endpointBuilder' +export type { Encoder, EncoderResult } from '../transport/encoder' +export { createIdentityEncoder } from '../transport/encoder' +export type { Payload, HttpResponse, BandwidthStats, HttpRequestEvent } from '../transport/payload' +export type { HttpRequest, SendStrategy, SendOnExitStrategy } from '../transport/httpRequest' +export { createHttpRequest } from '../transport/httpRequest' +export type { Batch } from '../transport/batch' +export { createBatch, MESSAGE_BYTES_LIMIT } from '../transport/batch' +export { RECOMMENDED_REQUEST_BYTES_LIMIT } from '../transport/payload' +export { PageExitReason, isPageExitReason, FLUSH_DURATION_LIMIT } from '../transport/flushController' +export type { PageMayExitEvent, UrgentFlushReason, FlushReason, FlushEvent } from '../transport/flushController' + export type { Site } from '../transport/intakeSites' export { INTAKE_SITE_STAGING, diff --git a/packages/js-core/src/entries/util.ts b/packages/js-core/src/entries/util.ts index f5446dc452..38f35dbc8b 100644 --- a/packages/js-core/src/entries/util.ts +++ b/packages/js-core/src/entries/util.ts @@ -4,6 +4,18 @@ export { setDebugMode, getDebugMode } from '../util/debug' export * from '../util/mergeInto' export * from '../util/typeUtils' export { globalObject, isWorkerEnvironment } from '../util/globalObject' +export * from '../util/byteUtils' +export type { Context, ContextValue, ContextArray } from '../util/context' +export * from '../util/jsonStringify' +export { objectValues } from '../util/polyfills' +export { isServerError } from '../util/responseUtils' +export { mockable, mockableReplacements } from '../util/mockable' +export type { BrowserWindowWithZoneJs } from '../util/getZoneJsOriginalValue' +export { getZoneJsOriginalValue } from '../util/getZoneJsOriginalValue' +export type { TimeoutId } from '../util/timer' +export { setTimeout, clearTimeout, setInterval, clearInterval } from '../util/timer' +export type { Subscription } from '../util/observable' +export { Observable, BufferedObservable, mergeObservables } from '../util/observable' export type { GlobalObject, Navigator, diff --git a/packages/browser-core/src/transport/batch.spec.ts b/packages/js-core/src/transport/batch.spec.ts similarity index 90% rename from packages/browser-core/src/transport/batch.spec.ts rename to packages/js-core/src/transport/batch.spec.ts index 4ab0d70ec9..15210a34d7 100644 --- a/packages/browser-core/src/transport/batch.spec.ts +++ b/packages/js-core/src/transport/batch.spec.ts @@ -1,14 +1,14 @@ -import { Observable } from '..' -import type { MockFlushController } from '../../test' -import { createMockFlushController, replaceMockable } from '../../test' -import { display } from '../tools/display' -import type { Encoder } from '../tools/encoder' -import { createIdentityEncoder } from '../tools/encoder' -import { createPageMayExitObservable } from '../browser/pageMayExitObservable' -import { createBatch, MESSAGE_BYTES_LIMIT, type Batch } from './batch' +import { Observable } from '../util/observable' +import { replaceMockable } from '../../test/replaceMockable' +import type { MockFlushController } from '../../test/mockFlushController' +import { createMockFlushController } from '../../test/mockFlushController' +import type { HttpRequestEvent } from './payload' +import type { Encoder } from './encoder' +import { createIdentityEncoder } from './encoder' import { createFlushController } from './flushController' -import { createHttpRequest } from './httpRequest' -import type { HttpRequest, HttpRequestEvent } from './httpRequest' +import { createBatch, MESSAGE_BYTES_LIMIT } from './batch' +import type { Batch } from './batch' +import type { HttpRequest } from './httpRequest' describe('batch', () => { const BIG_MESSAGE_OVER_BYTES_LIMIT = { message: 'x'.repeat(MESSAGE_BYTES_LIMIT + 1) } @@ -22,9 +22,9 @@ describe('batch', () => { send: jasmine.Spy sendOnExit: jasmine.Spy } - let flushController: MockFlushController let encoder: Encoder + let warnSpy: jasmine.Spy beforeEach(() => { transport = { @@ -34,10 +34,15 @@ describe('batch', () => { } satisfies HttpRequest flushController = createMockFlushController() encoder = createIdentityEncoder() - replaceMockable(createHttpRequest, (() => transport) as unknown as typeof createHttpRequest) - replaceMockable(createPageMayExitObservable, () => new Observable()) + warnSpy = jasmine.createSpy('warn') replaceMockable(createFlushController, () => flushController) - batch = createBatch({ encoder, endpoints: [], reportError: () => undefined }) + batch = createBatch({ + request: transport, + pageMayExitObservable: new Observable(), + encoder, + reportError: () => undefined, + warn: warnSpy, + }) }) it('should send a message', () => { @@ -94,7 +99,6 @@ describe('batch', () => { }) it('should not send a message with a bytes size above the limit', () => { - const warnSpy = spyOn(display, 'warn') batch.add(BIG_MESSAGE_OVER_BYTES_LIMIT) expect(warnSpy).toHaveBeenCalled() diff --git a/packages/js-core/src/transport/batch.ts b/packages/js-core/src/transport/batch.ts new file mode 100644 index 0000000000..d6b42812dc --- /dev/null +++ b/packages/js-core/src/transport/batch.ts @@ -0,0 +1,181 @@ +import { computeBytesCount, ONE_KIBI_BYTE } from '../util/byteUtils' +import type { Context } from '../util/context' +import { jsonStringify } from '../util/jsonStringify' +import { mockable } from '../util/mockable' +import type { Observable } from '../util/observable' +import { objectValues } from '../util/polyfills' +import type { EndpointBuilder } from './endpointBuilder' +import type { Encoder, EncoderResult } from './encoder' +import { createIdentityEncoder } from './encoder' +import { isPageExitReason, createFlushController } from './flushController' +import type { FlushEvent, FlushReason, UrgentFlushReason, PageMayExitEvent } from './flushController' +import type { HttpRequest } from './httpRequest' +import type { Payload } from './payload' + +/** Maximum byte size for a single serialised message. Messages exceeding this are discarded. */ +export const MESSAGE_BYTES_LIMIT = 256 * ONE_KIBI_BYTE + +/** + * The public handle returned by {@link createBatch}. + * + * Consumers add messages via {@link Batch.add} or {@link Batch.upsert} and the batch sends them + * automatically when a flush condition is met (size, count, time, or page exit). + */ +export interface Batch { + /** `true` when no messages are currently pending in the batch. */ + isEmpty: boolean + /** Appends a message to the batch. */ + add: (message: Context) => void + /** Inserts or replaces the message associated with `key`. */ + upsert: (message: Context, key: string) => void + /** Immediately flushes the batch for the given reason. */ + forceFlush: (reason: FlushReason) => void + /** Observable that fires just before an urgent (page-exit) flush so callers can finish pending work. */ + prepareUrgentFlushObservable: Observable + /** Observable that fires after each flush with metadata about the flushed batch. */ + flushObservable: Observable + /** Stops the batch and unsubscribes from internal observables. */ + stop: () => void +} + +/** + * Creates a batch that accumulates serialised messages and sends them to the intake endpoints + * via `request` whenever a flush condition is met. + * + * @param options - See parameter descriptions below. + * @param options.request - The HTTP request handle used to send payloads. + * @param options.pageMayExitObservable - Observable that signals imminent page unload; + * triggers an urgent flush to avoid data loss. + * @param options.endpoints - Intake endpoint builders (primary + optional replica). + * @param options.reportError - Called with a human-readable message when an internal error occurs. + * @param options.warn - Called to emit a warning when a message is discarded (e.g. too large). + * @param options.encoder - Encoder used to serialise message batches. + * Defaults to an identity (no-op) encoder. + */ +export function createBatch({ + request, + pageMayExitObservable, + reportError: _reportError, + warn, + encoder = createIdentityEncoder(), +}: { + request: HttpRequest + pageMayExitObservable: Observable + endpoints?: EndpointBuilder[] + reportError: (message: string) => void + warn: (message: string) => void + encoder?: Encoder +}): Batch { + const flushController = mockable(createFlushController)({ pageMayExitObservable }) + let upsertBuffer: { [key: string]: string } = {} + const flushSubscription = flushController.flushObservable.subscribe((event) => flush(event)) + + function push(serializedMessage: string, estimatedMessageBytesCount: number, key?: string) { + if (key !== undefined) { + let bytesDiff: number + if (upsertBuffer[key] !== undefined) { + bytesDiff = estimatedMessageBytesCount - encoder.estimateEncodedBytesCount(upsertBuffer[key]) + } else { + flushController.notifyBeforeAddMessage(estimatedMessageBytesCount) + bytesDiff = 0 + } + upsertBuffer[key] = serializedMessage + flushController.notifyAfterAddMessage(bytesDiff) + } else { + flushController.notifyBeforeAddMessage(estimatedMessageBytesCount) + encoder.write(encoder.isEmpty ? serializedMessage : `\n${serializedMessage}`, (realMessageBytesCount) => { + flushController.notifyAfterAddMessage(realMessageBytesCount - estimatedMessageBytesCount) + }) + } + } + + function addOrUpdate(message: Context, key?: string) { + const serializedMessage = jsonStringify(message)! + + const estimatedMessageBytesCount = encoder.estimateEncodedBytesCount(serializedMessage) + + if (estimatedMessageBytesCount >= MESSAGE_BYTES_LIMIT) { + warn( + `Discarded a message whose size was bigger than the maximum allowed size ${MESSAGE_BYTES_LIMIT / ONE_KIBI_BYTE}KiB.` + ) + return + } + + push(serializedMessage, estimatedMessageBytesCount, key) + } + + function flush(event: FlushEvent) { + const upsertMessages = objectValues(upsertBuffer).join('\n') + upsertBuffer = {} + + const pageMightExit = isPageExitReason(event.reason) + const send = pageMightExit ? request.sendOnExit : request.send + + if ( + pageMightExit && + // Note: checking that the encoder is async is not strictly needed, but it's an optimization: + // if the encoder is async we need to send two requests in some cases (one for encoded data + // and the other for non-encoded data). But if it's not async, we don't have to worry about + // it and always send a single request. + encoder.isAsync + ) { + const encoderResult = encoder.finishSync() + + // Send encoded messages + if (encoderResult.outputBytesCount) { + send(formatPayloadFromEncoder(encoderResult)) + } + + // Send messages that are not yet encoded at this point + const pendingMessages = [encoderResult.pendingData, upsertMessages].filter(Boolean).join('\n') + if (pendingMessages) { + send({ + data: pendingMessages, + bytesCount: computeBytesCount(pendingMessages), + }) + } + } else { + if (upsertMessages) { + encoder.write(encoder.isEmpty ? upsertMessages : `\n${upsertMessages}`) + } + encoder.finish((encoderResult) => { + send(formatPayloadFromEncoder(encoderResult)) + }) + } + } + + return { + get isEmpty() { + return flushController.messagesCount === 0 + }, + add: addOrUpdate, + upsert: addOrUpdate, + prepareUrgentFlushObservable: flushController.prepareUrgentFlushObservable, + forceFlush: flushController.forceFlush, + flushObservable: flushController.flushObservable, + stop: flushSubscription.unsubscribe, + } +} + +function formatPayloadFromEncoder(encoderResult: EncoderResult): Payload { + let data: string | Blob + if (typeof encoderResult.output === 'string') { + data = encoderResult.output + } else { + data = new Blob([encoderResult.output], { + // This will set the 'Content-Type: text/plain' header. Reasoning: + // * The intake rejects the request if there is no content type. + // * The browser will issue CORS preflight requests if we set it to 'application/json', which + // could induce higher intake load (and maybe has other impacts). + // * Also it's not quite JSON, since we are concatenating multiple JSON objects separated by + // new lines. + type: 'text/plain', + }) + } + + return { + data, + bytesCount: encoderResult.outputBytesCount, + encoding: encoderResult.encoding, + } +} diff --git a/packages/js-core/src/transport/encoder.ts b/packages/js-core/src/transport/encoder.ts new file mode 100644 index 0000000000..bc037e422c --- /dev/null +++ b/packages/js-core/src/transport/encoder.ts @@ -0,0 +1,139 @@ +import type { Uint8ArrayBuffer } from '../util/byteUtils' +import { computeBytesCount } from '../util/byteUtils' + +/** + * A generic encoding abstraction used by the batch transport layer to serialise + * outgoing payloads before sending them to the intake. + * + * Implementations may be synchronous (e.g. the identity encoder) or asynchronous + * (e.g. a deflate encoder backed by a Web Worker). Callers should check `isAsync` + * and handle both cases via `finish` / `finishSync` accordingly. + * + * @typeParam Output - The encoded output type: `string` for text encoders, + * `Uint8ArrayBuffer` for binary ones. + */ +export interface Encoder { + /** + * Whether this encoder may call `write` callbacks or `finish` callbacks asynchronously. + * When `false`, all callbacks are guaranteed to be invoked synchronously. + */ + isAsync: boolean + + /** + * `true` when no data has been written since the last `finish()` or `finishSync()` call. + */ + isEmpty: boolean + + /** + * Encodes `data` and appends it to the internal buffer. + * + * If provided, `callback` is called with the number of additional bytes added to the encoded + * output. For asynchronous encoders the callback may be deferred; it will not be called if + * `finish()` or `finishSync()` is invoked before encoding completes. + * + * @param data - The string to encode. + * @param callback - Optional callback receiving the encoded byte delta. + */ + write(data: string, callback?: (additionalEncodedBytesCount: number) => void): void + + /** + * Waits for any pending encodes and flushes the buffer, then invokes `callback` with the + * result. Resets the encoder state so it is ready for the next batch. + * + * For asynchronous encoders the callback may be deferred. It will not be called if another + * `finish()` or `finishSync()` call is made before encoding completes. + * + * @param callback - Called with the completed {@link EncoderResult}. + */ + finish(callback: (result: EncoderResult) => void): void + + /** + * Immediately flushes the buffer and returns the result, discarding any pending asynchronous + * encode operations and `finish()` callbacks. Resets the encoder state. + * + * @returns The {@link EncoderResult} along with any data that was still pending (not yet encoded) + * at the time of the call. + */ + finishSync(): EncoderResult & { pendingData: string } + + /** + * Returns a rough estimate of how many bytes `data` would occupy once encoded. + * Used to make batching decisions before the actual encoding is complete. + * + * @param data - The string to estimate. + * @returns Estimated encoded byte count. + */ + estimateEncodedBytesCount(data: string): number +} + +/** + * The result produced by {@link Encoder.finish} or {@link Encoder.finishSync}. + * + * @typeParam Output - The encoded output type: `string` or `Uint8ArrayBuffer`. + */ +export interface EncoderResult { + /** The encoded output. */ + output: Output + + /** Byte count of `output`. */ + outputBytesCount: number + + /** + * HTTP `Content-Encoding` value for the encoded data, if applicable. + * See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding#directives + */ + encoding?: 'deflate' + + /** Total byte count of the raw (pre-encoding) input strings, encoded as UTF-8. */ + rawBytesCount: number +} + +/** + * Creates a synchronous identity encoder that stores data as plain UTF-8 strings with no + * compression. + * + * Use this as the default encoder when no deflate / compression worker is available. + * + * @returns A synchronous {@link Encoder} whose `output` is always a `string`. + */ +export function createIdentityEncoder(): Encoder { + let output = '' + let outputBytesCount = 0 + + return { + isAsync: false, + + get isEmpty() { + return !output + }, + + write(data, callback) { + const additionalEncodedBytesCount = computeBytesCount(data) + outputBytesCount += additionalEncodedBytesCount + output += data + if (callback) { + callback(additionalEncodedBytesCount) + } + }, + + finish(callback) { + callback(this.finishSync()) + }, + + finishSync() { + const result = { + output, + outputBytesCount, + rawBytesCount: outputBytesCount, + pendingData: '', + } + output = '' + outputBytesCount = 0 + return result + }, + + estimateEncodedBytesCount(data) { + return data.length + }, + } +} diff --git a/packages/browser-core/src/transport/flushController.spec.ts b/packages/js-core/src/transport/flushController.spec.ts similarity index 96% rename from packages/browser-core/src/transport/flushController.spec.ts rename to packages/js-core/src/transport/flushController.spec.ts index 51912eca6e..a085505206 100644 --- a/packages/browser-core/src/transport/flushController.spec.ts +++ b/packages/js-core/src/transport/flushController.spec.ts @@ -1,10 +1,11 @@ -import type { Clock } from '../../test' -import { mockClock } from '../../test' -import type { PageMayExitEvent } from '../browser/pageMayExitObservable' -import { Observable } from '../tools/observable' -import type { FlushController, FlushEvent } from './flushController' +import type { Clock } from '../../test/mockClock' +import { mockClock } from '../../test/mockClock' +import { Observable } from '../util/observable' +import { RECOMMENDED_REQUEST_BYTES_LIMIT } from './payload' +import type { PageMayExitEvent, FlushEvent } from './flushController' import { createFlushController, FLUSH_DURATION_LIMIT, MESSAGES_LIMIT } from './flushController' -import { RECOMMENDED_REQUEST_BYTES_LIMIT } from './httpRequest' + +type FlushController = ReturnType const BYTES_LIMIT = RECOMMENDED_REQUEST_BYTES_LIMIT // Arbitrary message size that is below the BYTES_LIMIT diff --git a/packages/js-core/src/transport/flushController.ts b/packages/js-core/src/transport/flushController.ts new file mode 100644 index 0000000000..e25b4dacde --- /dev/null +++ b/packages/js-core/src/transport/flushController.ts @@ -0,0 +1,206 @@ +import type { Duration } from '../entries/time' +import { ONE_SECOND } from '../entries/time' +import { Observable } from '../util/observable' +import { isWorkerEnvironment } from '../util/globalObject' +import type { TimeoutId } from '../util/timer' +import { clearTimeout, setTimeout } from '../util/timer' +import { RECOMMENDED_REQUEST_BYTES_LIMIT } from './payload' + +/** + * The reason a page-exit flush was triggered. + * + * These string values are also recorded in RUM events, so they must not be + * changed without a corresponding intake schema update. + */ +export const PageExitReason = { + HIDDEN: 'visibility_hidden', + UNLOADING: 'before_unload', + PAGEHIDE: 'page_hide', + FROZEN: 'page_frozen', +} as const + +/** @see {@link PageExitReason} */ +export type PageExitReason = (typeof PageExitReason)[keyof typeof PageExitReason] + +/** Event emitted by a page-may-exit observable when the page is about to become inactive. */ +export interface PageMayExitEvent { + reason: PageExitReason +} + +/** + * Returns `true` when `reason` is a {@link PageExitReason} value. + * + * @param reason - Any string to test. + */ +export function isPageExitReason(reason: string): reason is PageExitReason { + return Object.values(PageExitReason).includes(reason as PageExitReason) +} + +/** + * A flush triggered by a page-exit signal — the most urgent kind because the + * page may unload before a normal flush cycle completes. + */ +export type UrgentFlushReason = PageExitReason + +/** + * All possible reasons a batch flush can be triggered. + * + * - Page-exit reasons (`UrgentFlushReason`) are the most urgent. + * - `'duration_limit'` fires after {@link FLUSH_DURATION_LIMIT} to keep ALB connections alive. + * - `'bytes_limit'` fires when the batch approaches the recommended request size. + * - `'messages_limit'` fires when the message count reaches {@link MESSAGES_LIMIT}. + * - `'session_expire'` fires when the SDK session ends. + */ +export type FlushReason = UrgentFlushReason | 'duration_limit' | 'bytes_limit' | 'messages_limit' | 'session_expire' + +/** + * Flush automatically — aim to stay below the ALB connection timeout to maximise connection reuse. + */ +export const FLUSH_DURATION_LIMIT = (30 * ONE_SECOND) as Duration + +/** + * Maximum number of messages per batch. + * + * In Worker environments the limit is 1 to ensure each batch fits in a single postMessage event. + */ +export const MESSAGES_LIMIT = isWorkerEnvironment ? 1 : 50 + +/** The inferred return type of {@link createFlushController}. */ +export type FlushController = ReturnType + +/** Payload emitted on `flushObservable` each time a batch is flushed. */ +export interface FlushEvent { + /** Why the flush was triggered. */ + reason: FlushReason + /** Total byte count of the messages in the flushed batch. */ + bytesCount: number + /** Number of messages in the flushed batch. */ + messagesCount: number +} + +interface FlushControllerOptions { + /** + * Observable that fires when the page is about to become inactive. + * Provided by callers so this module stays free of browser-specific code. + */ + pageMayExitObservable: Observable +} + +/** + * Returns a flush controller: an object responsible for deciding when a pool of + * pending messages should be sent to the intake. + * + * Flush can be triggered by: + * - page exit (urgent, via the injected `pageMayExitObservable`) + * - accumulated byte count reaching {@link RECOMMENDED_REQUEST_BYTES_LIMIT} + * - message count reaching {@link MESSAGES_LIMIT} + * - a periodic timer firing after {@link FLUSH_DURATION_LIMIT} + * - an explicit {@link FlushController.forceFlush} call + * + * The implementation supports both synchronous and asynchronous callers but + * relies on the invariants documented on each method to stay coherent. + * + * @param options - Configuration; see {@link FlushControllerOptions}. + * @param options.pageMayExitObservable - Observable signalling imminent page unload. + */ +export function createFlushController({ pageMayExitObservable }: FlushControllerOptions) { + let forcedFlushReason: FlushReason | undefined + const prepareUrgentFlushObservable = new Observable() + const pageMayExitSubscription = pageMayExitObservable.subscribe((event) => { + forcedFlushReason = event.reason + try { + prepareUrgentFlushObservable.notify(event.reason) + } finally { + forcedFlushReason = undefined + } + flush(event.reason) + }) + + const flushObservable = new Observable(() => () => { + pageMayExitSubscription.unsubscribe() + }) + + let currentBytesCount = 0 + let currentMessagesCount = 0 + + function flush(flushReason: FlushReason) { + if (currentMessagesCount === 0) { + return + } + + const messagesCount = currentMessagesCount + const bytesCount = currentBytesCount + + currentMessagesCount = 0 + currentBytesCount = 0 + cancelDurationLimitTimeout() + + flushObservable.notify({ + reason: flushReason, + messagesCount, + bytesCount, + }) + } + + let durationLimitTimeoutId: TimeoutId | undefined + function scheduleDurationLimitTimeout() { + if (durationLimitTimeoutId === undefined) { + durationLimitTimeoutId = setTimeout(() => { + flush('duration_limit') + }, FLUSH_DURATION_LIMIT) + } + } + + function cancelDurationLimitTimeout() { + clearTimeout(durationLimitTimeoutId) + durationLimitTimeoutId = undefined + } + + return { + flushObservable, + prepareUrgentFlushObservable, + forceFlush: flush, + get messagesCount() { + return currentMessagesCount + }, + + /** + * Notifies that a message is about to be added to the pending pool. + * + * Must be called synchronously, immediately before adding the message, so + * that no flush event can occur between this call and the actual addition. + * + * @param estimatedMessageBytesCount - Estimated byte size of the message once added. + */ + notifyBeforeAddMessage(estimatedMessageBytesCount: number) { + if (currentBytesCount + estimatedMessageBytesCount >= RECOMMENDED_REQUEST_BYTES_LIMIT) { + flush(forcedFlushReason ?? 'bytes_limit') + } + // Consider the message to be added now rather than in `notifyAfterAddMessage`, because if no + // message was added yet and `notifyAfterAddMessage` is called asynchronously, we still want + // to notify when a flush is needed (for example on page exit). + currentMessagesCount += 1 + currentBytesCount += estimatedMessageBytesCount + scheduleDurationLimitTimeout() + }, + + /** + * Notifies that a message was added to the pending pool. + * + * May be called asynchronously after the addition, but must not be called if + * a flush occurred between the addition and this call. + * + * @param messageBytesCountDiff - Difference between the estimated and actual byte size. + * Defaults to `0` when the estimate was exact. + */ + notifyAfterAddMessage(messageBytesCountDiff = 0) { + currentBytesCount += messageBytesCountDiff + + if (currentMessagesCount >= MESSAGES_LIMIT) { + flush(forcedFlushReason ?? 'messages_limit') + } else if (currentBytesCount >= RECOMMENDED_REQUEST_BYTES_LIMIT) { + flush(forcedFlushReason ?? 'bytes_limit') + } + }, + } +} diff --git a/packages/js-core/src/transport/httpRequest.ts b/packages/js-core/src/transport/httpRequest.ts new file mode 100644 index 0000000000..84f52fe885 --- /dev/null +++ b/packages/js-core/src/transport/httpRequest.ts @@ -0,0 +1,89 @@ +import { Observable } from '../util/observable' +import type { EndpointBuilder } from './endpointBuilder' +import { newRetryState, sendWithRetryStrategy } from './sendWithRetryStrategy' +import type { Payload, HttpResponse, HttpRequestEvent } from './payload' + +/** + * A send strategy used by {@link createHttpRequest} for normal (non-exit) requests. + * + * @typeParam Body - The payload type. + */ +export type SendStrategy = ( + endpointBuilder: EndpointBuilder, + payload: Body, + onResponse: (response: HttpResponse) => void +) => void + +/** + * A send strategy used by {@link createHttpRequest} for page-exit requests. + * + * Unlike {@link SendStrategy}, there is no `onResponse` callback: page-exit sends are + * best-effort and the page may unload before a response arrives. + * + * @typeParam Body - The payload type. + */ +export type SendOnExitStrategy = ( + endpointBuilder: EndpointBuilder, + payload: Body +) => void + +/** + * An HTTP request handle returned by {@link createHttpRequest}. + * + * @typeParam Body - The payload type, defaulting to {@link Payload}. + */ +export interface HttpRequest { + /** Observable that emits an event for every send attempt outcome. */ + observable: Observable> + /** Sends `payload` using the normal send strategy, with retry on transient failure. */ + send(this: void, payload: Body): void + /** Sends `payload` using the page-exit send strategy (best-effort, no retry). */ + sendOnExit(this: void, payload: Body): void +} + +/** + * Creates an {@link HttpRequest} that sends payloads to every endpoint in `endpointBuilders`, + * with automatic retry on transient failures. + * + * Browser-specific send mechanisms (fetch, sendBeacon) are injected via `sendStrategy` and + * `sendOnExitStrategy` so this function remains free of browser APIs. + * + * @param endpointBuilders - Intake endpoints to send to (typically one primary + optional replica). + * @param reportError - Called with a human-readable message when the send queue overflows. + * @param sendStrategy - How to send a normal (non-exit) request. + * @param sendOnExitStrategy - How to send a request on page exit (best-effort). + */ +export function createHttpRequest( + endpointBuilders: EndpointBuilder[], + reportError: (message: string) => void, + sendStrategy: SendStrategy, + sendOnExitStrategy: SendOnExitStrategy +): HttpRequest { + const observable = new Observable>() + const retryState = newRetryState() + + return { + observable, + send: (payload: Body) => { + for (const endpointBuilder of endpointBuilders) { + sendWithRetryStrategy( + payload, + retryState, + (payload, onResponse) => sendStrategy(endpointBuilder, payload, onResponse), + endpointBuilder.trackType, + reportError, + observable + ) + } + }, + /** + * Since fetch keepalive behaves like regular fetch on Firefox, + * keep using sendBeaconStrategy on exit. + */ + sendOnExit: (payload: Body) => { + for (const endpointBuilder of endpointBuilders) { + sendOnExitStrategy(endpointBuilder, payload) + } + }, + } +} diff --git a/packages/js-core/src/transport/payload.ts b/packages/js-core/src/transport/payload.ts new file mode 100644 index 0000000000..acaf4ed7e9 --- /dev/null +++ b/packages/js-core/src/transport/payload.ts @@ -0,0 +1,72 @@ +import { ONE_KIBI_BYTE } from '../util/byteUtils' +import type { Context } from '../util/context' +import type { TransportRetryInfo } from './endpointBuilder' + +/** + * Maximum recommended byte count for a single HTTP request body. + * + * Beacon payloads are capped at 64 KiB by the browser; we leave room for + * logs, RUM and other consumers by using 16 KiB as the recommended limit. + * Used both to trigger batch flushes and as the default `sendBeacon` size guard. + */ +export const RECOMMENDED_REQUEST_BYTES_LIMIT = 16 * ONE_KIBI_BYTE + +/** + * A unit of data ready to be sent to an intake endpoint. + * + * `data` is the serialised (and optionally encoded) body. `bytesCount` is the + * byte size of `data` as it will be transmitted — used for bandwidth accounting + * and queue-full decisions. + */ +export interface Payload { + data: string | FormData | Blob + bytesCount: number + retry?: TransportRetryInfo + encoding?: 'deflate' +} + +/** + * An HTTP response received for an intake request. + * + * Extends {@link Context} so that extra metadata can be carried alongside the + * mandatory `status` field. + */ +export interface HttpResponse extends Context { + status: number + type?: ResponseType +} + +/** + * Point-in-time bandwidth counters for all ongoing intake requests. + */ +export interface BandwidthStats { + /** Total byte count currently in-flight across all ongoing requests. */ + ongoingByteCount: number + /** Number of requests currently in flight. */ + ongoingRequestCount: number +} + +/** + * An event emitted on the {@link HttpRequest} observable to report the outcome of a send attempt. + * + * @typeParam Body - The payload type, defaulting to {@link Payload}. + */ +export type HttpRequestEvent = + | { + /** A request to send the given payload failed. (We may retry.) */ + type: 'failure' + bandwidth: BandwidthStats + payload: Body + } + | { + /** The given payload was discarded because the request queue is full. */ + type: 'queue-full' + bandwidth: BandwidthStats + payload: Body + } + | { + /** A request to send the given payload succeeded. */ + type: 'success' + bandwidth: BandwidthStats + payload: Body + } diff --git a/packages/browser-core/src/transport/sendWithRetryStrategy.spec.ts b/packages/js-core/src/transport/sendWithRetryStrategy.spec.ts similarity index 98% rename from packages/browser-core/src/transport/sendWithRetryStrategy.spec.ts rename to packages/js-core/src/transport/sendWithRetryStrategy.spec.ts index f14cd74250..a225ae7bad 100644 --- a/packages/browser-core/src/transport/sendWithRetryStrategy.spec.ts +++ b/packages/js-core/src/transport/sendWithRetryStrategy.spec.ts @@ -1,7 +1,9 @@ -import { mockClock, setNavigatorOnLine } from '../../test' -import type { Clock } from '../../test' -import { Observable } from '../tools/observable' -import { ONE_MEBI_BYTE } from '../tools/utils/byteUtils' +import type { Clock } from '../../test/mockClock' +import { mockClock } from '../../test/mockClock' +import { setNavigatorOnLine } from '../../test/mockNavigator' +import { Observable } from '../util/observable' +import { ONE_MEBI_BYTE } from '../util/byteUtils' +import type { Payload, HttpResponse, HttpRequestEvent } from './payload' import type { RetryState } from './sendWithRetryStrategy' import { newRetryState, @@ -11,7 +13,6 @@ import { MAX_QUEUE_BYTES_COUNT, INITIAL_BACKOFF_TIME, } from './sendWithRetryStrategy' -import type { Payload, HttpResponse, HttpRequestEvent } from './httpRequest' describe('sendWithRetryStrategy', () => { const ENDPOINT_TYPE = 'logs' diff --git a/packages/browser-core/src/transport/sendWithRetryStrategy.ts b/packages/js-core/src/transport/sendWithRetryStrategy.ts similarity index 81% rename from packages/browser-core/src/transport/sendWithRetryStrategy.ts rename to packages/js-core/src/transport/sendWithRetryStrategy.ts index 70488b29f3..10504820a7 100644 --- a/packages/browser-core/src/transport/sendWithRetryStrategy.ts +++ b/packages/js-core/src/transport/sendWithRetryStrategy.ts @@ -1,16 +1,18 @@ -import { ONE_MINUTE, ONE_SECOND } from '@datadog/js-core/time' -import type { TrackType } from '@datadog/js-core/transport' -import { setTimeout } from '../tools/timer' -import { ONE_MEBI_BYTE, ONE_KIBI_BYTE } from '../tools/utils/byteUtils' -import { isServerError } from '../tools/utils/responseUtils' -import type { Observable } from '../tools/observable' -import type { Payload, HttpRequestEvent, HttpResponse, BandwidthStats } from './httpRequest' +import type { Duration } from '../entries/time' +import { ONE_MINUTE, ONE_SECOND } from '../entries/time' +import { setTimeout } from '../util/timer' +import { ONE_MEBI_BYTE, ONE_KIBI_BYTE } from '../util/byteUtils' +import { isServerError } from '../util/responseUtils' +import { globalObject } from '../util/globalObject' +import type { Observable } from '../util/observable' +import type { TrackType } from './endpointBuilder' +import type { Payload, HttpRequestEvent, HttpResponse, BandwidthStats } from './payload' export const MAX_ONGOING_BYTES_COUNT = 80 * ONE_KIBI_BYTE export const MAX_ONGOING_REQUESTS = 32 export const MAX_QUEUE_BYTES_COUNT = 20 * ONE_MEBI_BYTE -export const MAX_BACKOFF_TIME = ONE_MINUTE -export const INITIAL_BACKOFF_TIME = ONE_SECOND +export const MAX_BACKOFF_TIME = ONE_MINUTE as Duration +export const INITIAL_BACKOFF_TIME = ONE_SECOND as Duration const enum TransportStatus { UP, @@ -23,6 +25,7 @@ const enum RetryReason { AFTER_RESUME, } +/** Internal retry state held by each {@link createHttpRequest} instance. */ export interface RetryState { transportStatus: TransportStatus currentBackoffTime: number @@ -33,6 +36,17 @@ export interface RetryState { type SendStrategy = (payload: Body, onResponse: (r: HttpResponse) => void) => void +/** + * Sends `payload` via `sendStrategy`, automatically retrying on transient failures with + * exponential back-off. + * + * @param payload - The payload to send. + * @param state - Mutable retry state shared across calls for the same track type. + * @param sendStrategy - The function that performs the actual HTTP request. + * @param trackType - The intake track being targeted (used in error messages). + * @param reportError - Called with a human-readable message when the queue overflows. + * @param requestObservable - Observable notified with every request outcome. + */ export function sendWithRetryStrategy( payload: Body, state: RetryState, @@ -139,13 +153,18 @@ function retryQueuedPayloads( function shouldRetryRequest(response: HttpResponse) { return ( response.type !== 'opaque' && - ((response.status === 0 && !navigator.onLine) || + ((response.status === 0 && !globalObject.navigator?.onLine) || response.status === 408 || response.status === 429 || isServerError(response.status)) ) } +/** + * Creates a fresh {@link RetryState} for a new intake track connection. + * + * @returns An initialised {@link RetryState} ready for use with {@link sendWithRetryStrategy}. + */ export function newRetryState(): RetryState { return { transportStatus: TransportStatus.UP, diff --git a/packages/js-core/src/util/byteUtils.ts b/packages/js-core/src/util/byteUtils.ts new file mode 100644 index 0000000000..4bd3f8b1a8 --- /dev/null +++ b/packages/js-core/src/util/byteUtils.ts @@ -0,0 +1,64 @@ +/** One kibibyte in bytes (1024). */ +export const ONE_KIBI_BYTE = 1024 + +/** One mebibyte in bytes (1024 × 1024). */ +export const ONE_MEBI_BYTE = 1024 * ONE_KIBI_BYTE + +// eslint-disable-next-line no-control-regex +const HAS_MULTI_BYTES_CHARACTERS = /[^\u0000-\u007F]/ + +/** + * A `Uint8Array` whose underlying storage is a plain `ArrayBuffer`. + * + * This is a stricter subtype of `Uint8Array` that guarantees `.buffer` is always an + * `ArrayBuffer` (not a `SharedArrayBuffer`), making it safe to pass to APIs that require + * an owned `ArrayBuffer`. + */ +export interface Uint8ArrayBuffer extends Uint8Array { + readonly buffer: ArrayBuffer + + subarray(begin?: number, end?: number): Uint8ArrayBuffer +} + +/** + * Computes the byte count of a string when encoded as UTF-8. + * + * Uses `TextEncoder` for accuracy when multi-byte characters are present, and falls back to + * the string's `.length` property for ASCII-only strings as a performance optimisation. + * + * @param candidate - The string whose byte count to compute. + * @returns The number of bytes needed to encode `candidate` as UTF-8. + */ +export function computeBytesCount(candidate: string): number { + // Accurate bytes count computations can degrade performances when there is a lot of events to process + if (!HAS_MULTI_BYTES_CHARACTERS.test(candidate)) { + return candidate.length + } + + return new TextEncoder().encode(candidate).length +} + +/** + * Concatenates multiple `Uint8ArrayBuffer` instances into a single contiguous buffer. + * + * As a performance optimisation, returns the input buffer directly when the array contains + * exactly one element (avoiding an unnecessary copy). + * + * @param buffers - The buffers to concatenate, in order. + * @returns A new `Uint8ArrayBuffer` containing all input buffers joined end-to-end. + */ +export function concatBuffers(buffers: Uint8ArrayBuffer[]): Uint8ArrayBuffer { + // Optimization: if there is a single buffer, no need to copy it + if (buffers.length === 1) { + return buffers[0] + } + + const length = buffers.reduce((total, buffer) => total + buffer.length, 0) + const result: Uint8ArrayBuffer = new Uint8Array(length) + let offset = 0 + for (const buffer of buffers) { + result.set(buffer, offset) + offset += buffer.length + } + return result +} diff --git a/packages/js-core/src/util/context.ts b/packages/js-core/src/util/context.ts new file mode 100644 index 0000000000..483cd4fd7b --- /dev/null +++ b/packages/js-core/src/util/context.ts @@ -0,0 +1,22 @@ +/** + * An arbitrary JSON-serialisable object used throughout the SDK to carry event attributes, + * user-defined properties, and other structured metadata. + */ +export interface Context { + [x: string]: ContextValue +} + +/** + * A value that can appear anywhere in a {@link Context} tree: scalars, nested objects, arrays, + * or absent/null values. + */ +export type ContextValue = string | number | boolean | Context | ContextArray | undefined | null + +/** + * An array of {@link ContextValue} items. Defined as a named interface (rather than an inline + * `Array`) to allow recursive references in the {@link ContextValue} union. + * + * @hidden + */ +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +export interface ContextArray extends Array {} diff --git a/packages/js-core/src/util/getZoneJsOriginalValue.ts b/packages/js-core/src/util/getZoneJsOriginalValue.ts new file mode 100644 index 0000000000..9a6d51e6a0 --- /dev/null +++ b/packages/js-core/src/util/getZoneJsOriginalValue.ts @@ -0,0 +1,50 @@ +import { globalObject } from './globalObject' + +/** + * A global object that may carry Zone.js's symbol-based original-value storage. + * + * Zone.js patches many browser APIs and stores the originals under a hidden property whose + * name is generated by `Zone.__symbol__`. + */ +export interface BrowserWindowWithZoneJs { + Zone?: { + /** + * Returns the hidden property name Zone.js uses to store the original (unpatched) value + * for a given API name. Not all Zone.js versions expose this — treat it as optional. + */ + __symbol__?: (name: string) => string + } +} + +/** + * Returns the original (pre-Zone.js-patch) value of a property on `target`. + * + * Zone.js[1] patches many browser and JS platform APIs (e.g. `setTimeout`, `fetch`, + * `addEventListener`) and stores the originals under a hidden property whose name is derived from + * `Zone.__symbol__(name)`. In multiple cases Zone.js has been identified as the root cause of + * serious issues such as memory leaks and high CPU usage. Using the original implementation + * avoids those pitfalls. + * + * If Zone.js is not present, or if it has not patched `name`, the property is read directly from + * `target` as a fallback. + * + * [1]: https://github.com/angular/angular/tree/main/packages/zone.js + * + * @param target - The object whose property should be retrieved (e.g. `globalObject`). + * @param name - The property name to look up (e.g. `'setTimeout'`). + * @returns The original unpatched value of `target[name]`. + */ +export function getZoneJsOriginalValue( + target: Target, + name: Name +): Target[Name] { + const browserWindow = globalObject as BrowserWindowWithZoneJs + let original: Target[Name] | undefined + if (browserWindow.Zone && typeof browserWindow.Zone.__symbol__ === 'function') { + original = (target as any)[browserWindow.Zone.__symbol__(name)] + } + if (!original) { + original = target[name] + } + return original +} diff --git a/packages/js-core/src/util/jsonStringify.ts b/packages/js-core/src/util/jsonStringify.ts new file mode 100644 index 0000000000..47235b74f2 --- /dev/null +++ b/packages/js-core/src/util/jsonStringify.ts @@ -0,0 +1,72 @@ +// eslint-disable-next-line @typescript-eslint/no-empty-function +function noop() {} + +/** + * Custom implementation of `JSON.stringify` that ignores `toJSON` methods on the value and its + * prototype chain. + * + * Some sites badly override `toJSON` on built-in prototypes (e.g. `Object.prototype` or + * `Array.prototype`), which would corrupt the serialised output. Removing all `toJSON` methods + * from nested values would be too costly, so we only detach them from the root value and from + * `Object` / `Array` prototypes before delegating to the native `JSON.stringify`. + * + * @param value - The value to serialise. + * @param replacer - An optional array of allowed property names (same as `JSON.stringify`). + * @param space - An optional indent width or string (same as `JSON.stringify`). + * @returns The JSON string, or `undefined` for non-object primitives that `JSON.stringify` would + * return `undefined` for. + */ +export function jsonStringify( + value: unknown, + replacer?: Array, + space?: string | number +): string | undefined { + if (typeof value !== 'object' || value === null) { + return JSON.stringify(value) + } + + // Note: The order matters here. We need to detach toJSON methods on parent classes before their + // subclasses. + const restoreObjectPrototypeToJson = detachToJsonMethod(Object.prototype) + const restoreArrayPrototypeToJson = detachToJsonMethod(Array.prototype) + const restoreValuePrototypeToJson = detachToJsonMethod(Object.getPrototypeOf(value)) + const restoreValueToJson = detachToJsonMethod(value) + + try { + return JSON.stringify(value, replacer, space) + } catch { + return '' + } finally { + restoreObjectPrototypeToJson() + restoreArrayPrototypeToJson() + restoreValuePrototypeToJson() + restoreValueToJson() + } +} + +/** An object that may have a custom `toJSON` serialisation method. */ +export interface ObjectWithToJsonMethod { + toJSON?: () => unknown +} + +/** + * Removes the `toJSON` method from an object if present and returns a function that restores it. + * + * Used by {@link jsonStringify} to temporarily neutralise overridden `toJSON` methods on + * prototypes and values before delegating to the native `JSON.stringify`. + * + * @param value - The object from which to detach `toJSON`. + * @returns A zero-argument function that re-attaches the original `toJSON` (or a no-op if the + * object had none). + */ +export function detachToJsonMethod(value: object) { + const object = value as ObjectWithToJsonMethod + const objectToJson = object.toJSON + if (objectToJson) { + delete object.toJSON + return () => { + object.toJSON = objectToJson + } + } + return noop +} diff --git a/packages/js-core/src/util/mockable.ts b/packages/js-core/src/util/mockable.ts new file mode 100644 index 0000000000..6641f4c4dc --- /dev/null +++ b/packages/js-core/src/util/mockable.ts @@ -0,0 +1,47 @@ +declare const __BUILD_ENV__SDK_VERSION__: string + +/** + * A registry of test-time mock replacements, keyed by the original value. + * + * In production builds this map is never populated. In test builds, + * {@link mockable} reads from it to return the registered replacement instead + * of the real value. + * + * Exposed so that test helpers (e.g. `replaceMockable`) can register and clean + * up replacements without coupling them to this module's implementation. + */ +export const mockableReplacements = new Map() + +/** + * Wraps a value to make it replaceable in tests without changing its type. + * + * In production builds this is a no-op that returns `value` as-is. In test + * builds it checks {@link mockableReplacements} and returns the registered + * replacement if one exists, otherwise falls through to the real value. + * + * @param value - The real value (a function, object, or primitive) to wrap. + * @returns The replacement registered for `value` in test builds, or `value` + * itself in production and when no replacement is registered. + * @example + * // In source file: + * import { mockable } from '@datadog/js-core/util' + * export function formatNavigationEntry(): string { + * const navigationEntry = mockable(getNavigationEntry)() + * ... + * } + * + * // In test file: + * import { replaceMockable } from '@datadog/browser-core/test' + * it('...', () => { + * replaceMockable(getNavigationEntry, () => FAKE_NAVIGATION_ENTRY) + * expect(formatNavigationEntry()).toEqual(...) + * }) + */ +export function mockable(value: T): T { + // In test builds, return a wrapper that checks for mocks at call time + if (__BUILD_ENV__SDK_VERSION__ === 'test' && mockableReplacements.has(value)) { + return mockableReplacements.get(value)! as T + } + // In production, return the value as-is + return value +} diff --git a/packages/js-core/src/util/observable.ts b/packages/js-core/src/util/observable.ts new file mode 100644 index 0000000000..4601b5d46a --- /dev/null +++ b/packages/js-core/src/util/observable.ts @@ -0,0 +1,152 @@ +import { queueMicrotask } from './queueMicrotask' + +/** A handle returned by {@link Observable.subscribe} that allows cancelling the subscription. */ +export interface Subscription { + unsubscribe: () => void +} + +type Observer = (data: T) => void + +/** + * A minimal observable / event-emitter that supports subscribing, unsubscribing, and notifying + * observers synchronously. + * + * An optional `onFirstSubscribe` callback can be provided to the constructor to perform setup work + * (e.g. attaching a DOM listener) lazily — it runs when the first observer subscribes and may + * return a teardown function that is called when the last observer unsubscribes. + * + * @typeParam T - The type of value emitted to observers. + */ +// eslint-disable-next-line no-restricted-syntax +export class Observable { + protected observers: Array> = [] + private onLastUnsubscribe?: () => void + + constructor(private onFirstSubscribe?: (observable: Observable) => (() => void) | void) {} + + /** Adds `observer` and returns a {@link Subscription} that removes it. */ + subscribe(observer: Observer): Subscription { + this.addObserver(observer) + return { + unsubscribe: () => this.removeObserver(observer), + } + } + + /** Synchronously calls every currently subscribed observer with `data`. */ + notify(data: T) { + this.observers.forEach((observer) => observer(data)) + } + + protected addObserver(observer: Observer) { + this.observers.push(observer) + if (this.observers.length === 1 && this.onFirstSubscribe) { + this.onLastUnsubscribe = this.onFirstSubscribe(this) || undefined + } + } + + protected removeObserver(observer: Observer) { + this.observers = this.observers.filter((other) => observer !== other) + if (!this.observers.length && this.onLastUnsubscribe) { + this.onLastUnsubscribe() + } + } +} + +/** + * Merges multiple observables into a single observable that emits whenever any source emits. + * + * Subscribing to the merged observable subscribes to all sources; unsubscribing from it + * unsubscribes from all sources. + * + * @param observables - The source observables to merge. + * @returns A new {@link Observable} that forwards emissions from all sources. + */ +export function mergeObservables(...observables: Array>) { + return new Observable((globalObservable) => { + const subscriptions: Subscription[] = observables.map((observable) => + observable.subscribe((data) => globalObservable.notify(data)) + ) + return () => subscriptions.forEach((subscription) => subscription.unsubscribe()) + }) +} + +/** + * An {@link Observable} that buffers emitted values and replays them to late subscribers. + * + * When a new observer subscribes, all values buffered since the last subscriber was added are + * delivered to it asynchronously (via {@link queueMicrotask}) before the observer is added to the + * live set. This avoids re-entrant notifications during subscription. + * + * The buffer is bounded by `maxBufferSize`; when it overflows, the oldest entry is dropped and + * `onDrop` is called with the total number of dropped items once the buffer is cleared. + * + * @typeParam T - The type of value buffered and emitted. + */ +// eslint-disable-next-line no-restricted-syntax +export class BufferedObservable extends Observable { + private buffer: T[] = [] + private droppedCount = 0 + + /** + * Creates a new `BufferedObservable`. + * + * @param maxBufferSize - Maximum number of values to keep in the buffer. + * @param onDrop - Optional callback invoked with the total drop count when the buffer is cleared. + */ + constructor( + private maxBufferSize: number, + private onDrop?: (count: number) => void + ) { + super() + } + + notify(data: T) { + this.buffer.push(data) + if (this.buffer.length > this.maxBufferSize) { + this.buffer.shift() + this.droppedCount++ + } + super.notify(data) + } + + subscribe(observer: Observer): Subscription { + let closed = false + + const subscription = { + unsubscribe: () => { + closed = true + this.removeObserver(observer) + }, + } + + queueMicrotask(() => { + for (const data of this.buffer) { + if (closed) { + return + } + observer(data) + } + + if (!closed) { + this.addObserver(observer) + } + }) + + return subscription + } + + /** + * Drops all buffered data and disables future buffering. + * + * Call this when the buffer is no longer needed to free memory and clarify intent. + * If items were dropped, `onDrop` is called once asynchronously (via a microtask). + */ + unbuffer() { + queueMicrotask(() => { + if (this.droppedCount > 0 && this.onDrop) { + this.onDrop(this.droppedCount) + } + this.maxBufferSize = this.buffer.length = 0 + }) + } +} diff --git a/packages/js-core/src/util/polyfills.ts b/packages/js-core/src/util/polyfills.ts new file mode 100644 index 0000000000..18ef5e4a1b --- /dev/null +++ b/packages/js-core/src/util/polyfills.ts @@ -0,0 +1,12 @@ +/** + * Returns the values of an object as an array, equivalent to `Object.values(object)`. + * + * Defined as a named wrapper rather than an inline `Object.values` call so that bundlers can + * mangle the property name, reducing bundle size when the function is called in many places. + * + * @param object - The object whose enumerable own property values to return. + * @returns An array of the object's values. + */ +export function objectValues(object: { [key: string]: T }) { + return Object.values(object) +} diff --git a/packages/js-core/src/util/queueMicrotask.ts b/packages/js-core/src/util/queueMicrotask.ts new file mode 100644 index 0000000000..3775a3ed44 --- /dev/null +++ b/packages/js-core/src/util/queueMicrotask.ts @@ -0,0 +1,25 @@ +import { globalObject } from './globalObject' + +/** + * Schedules `callback` to run as a microtask, using the native `queueMicrotask` when available + * and falling back to `Promise.resolve().then(callback)` otherwise. + * + * The native implementation is looked up on `globalObject` at call-time to support environments + * where the global object changes after module initialisation (e.g. Selenium GeckoDriver's + * `executeScript`). Binding it early with `.bind(globalObject)` throws + * `"queueMicrotask called on an object that does not implement interface Window"` in those + * environments; calling it as an unbound method avoids the issue. + * See https://github.com/mozilla/geckodriver/issues/1798 + * + * @param callback - The function to schedule as a microtask. + */ +export function queueMicrotask(callback: () => void) { + const nativeImplementation = globalObject.queueMicrotask + + if (typeof nativeImplementation === 'function') { + nativeImplementation(callback) + } else { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + Promise.resolve().then(callback) + } +} diff --git a/packages/js-core/src/util/responseUtils.ts b/packages/js-core/src/util/responseUtils.ts new file mode 100644 index 0000000000..516a20e5d9 --- /dev/null +++ b/packages/js-core/src/util/responseUtils.ts @@ -0,0 +1,11 @@ +/** + * Returns `true` when an HTTP status code indicates a server-side error (5xx). + * + * Used by the retry strategy to decide whether a failed request should be retried. + * + * @param status - The HTTP response status code to check. + * @returns `true` if `status` is 500 or above, `false` otherwise. + */ +export function isServerError(status: number) { + return status >= 500 +} diff --git a/packages/js-core/src/util/timer.ts b/packages/js-core/src/util/timer.ts new file mode 100644 index 0000000000..4c009c4239 --- /dev/null +++ b/packages/js-core/src/util/timer.ts @@ -0,0 +1,54 @@ +import type { GlobalObject } from './globalObject' +import { globalObject } from './globalObject' +import { getZoneJsOriginalValue } from './getZoneJsOriginalValue' + +/** + * The return type of a `setTimeout` or `setInterval` call. + * + * Defined as a named alias so it can be used as an opaque handle without committing to a + * concrete platform type (`number` in browsers, `NodeJS.Timeout` in Node.js). + */ +export type TimeoutId = ReturnType + +/** + * Zone.js-safe wrapper for `setTimeout`. + * + * Looks up the original `setTimeout` via {@link getZoneJsOriginalValue} to bypass any Zone.js + * patch, preventing the resource-exhaustion issues that Zone.js-patched timers can cause. + * + * @param callback - Function to invoke after `delay` milliseconds. + * @param delay - Delay in milliseconds (defaults to 0). + * @returns A {@link TimeoutId} that can be passed to {@link clearTimeout}. + */ +export function setTimeout(callback: () => void, delay?: number): TimeoutId { + return getZoneJsOriginalValue(globalObject, 'setTimeout')(callback, delay) +} + +/** + * Zone.js-safe wrapper for `clearTimeout`. + * + * @param timeoutId - The id returned by {@link setTimeout}, or `undefined` (no-op). + */ +export function clearTimeout(timeoutId: TimeoutId | undefined) { + getZoneJsOriginalValue(globalObject, 'clearTimeout')(timeoutId) +} + +/** + * Zone.js-safe wrapper for `setInterval`. + * + * @param callback - Function to invoke every `delay` milliseconds. + * @param delay - Interval in milliseconds (defaults to 0). + * @returns A {@link TimeoutId} that can be passed to {@link clearInterval}. + */ +export function setInterval(callback: () => void, delay?: number): TimeoutId { + return getZoneJsOriginalValue(globalObject, 'setInterval')(callback, delay) +} + +/** + * Zone.js-safe wrapper for `clearInterval`. + * + * @param timeoutId - The id returned by {@link setInterval}, or `undefined` (no-op). + */ +export function clearInterval(timeoutId: TimeoutId | undefined) { + getZoneJsOriginalValue(globalObject, 'clearInterval')(timeoutId) +} diff --git a/packages/js-core/test/mockClock.ts b/packages/js-core/test/mockClock.ts new file mode 100644 index 0000000000..8b74c61674 --- /dev/null +++ b/packages/js-core/test/mockClock.ts @@ -0,0 +1,12 @@ +import { registerCleanupTask } from './registerCleanupTask' + +export type Clock = ReturnType + +export function mockClock() { + jasmine.clock().install() + jasmine.clock().mockDate() + registerCleanupTask(() => jasmine.clock().uninstall()) + return { + tick: (ms: number) => jasmine.clock().tick(ms), + } +} diff --git a/packages/browser-core/test/emulate/mockFlushController.ts b/packages/js-core/test/mockFlushController.ts similarity index 87% rename from packages/browser-core/test/emulate/mockFlushController.ts rename to packages/js-core/test/mockFlushController.ts index c6a158c6c9..c27f795195 100644 --- a/packages/browser-core/test/emulate/mockFlushController.ts +++ b/packages/js-core/test/mockFlushController.ts @@ -1,5 +1,12 @@ -import { Observable } from '../../src/tools/observable' -import type { FlushEvent, FlushController, FlushReason, UrgentFlushReason } from '../../src/transport' +import { Observable } from '../src/util/observable' +import type { + createFlushController, + FlushEvent, + FlushReason, + UrgentFlushReason, +} from '../src/transport/flushController' + +type FlushController = ReturnType export type MockFlushController = ReturnType diff --git a/packages/js-core/test/mockNavigator.ts b/packages/js-core/test/mockNavigator.ts new file mode 100644 index 0000000000..a769cfea5d --- /dev/null +++ b/packages/js-core/test/mockNavigator.ts @@ -0,0 +1,13 @@ +import { registerCleanupTask } from './registerCleanupTask' + +export function setNavigatorOnLine(onLine: boolean) { + Object.defineProperty(navigator, 'onLine', { + get() { + return onLine + }, + configurable: true, + }) + registerCleanupTask(() => { + delete (navigator as any).onLine + }) +} diff --git a/packages/js-core/test/registerCleanupTask.ts b/packages/js-core/test/registerCleanupTask.ts new file mode 100644 index 0000000000..eaf1f8d4ed --- /dev/null +++ b/packages/js-core/test/registerCleanupTask.ts @@ -0,0 +1,13 @@ +type CleanupTask = () => unknown + +const cleanupTasks: CleanupTask[] = [] + +export function registerCleanupTask(task: CleanupTask) { + cleanupTasks.unshift(task) +} + +afterEach(async () => { + for (const task of cleanupTasks.splice(0)) { + await task() + } +}) diff --git a/packages/js-core/test/replaceMockable.ts b/packages/js-core/test/replaceMockable.ts new file mode 100644 index 0000000000..cd4c08e9ea --- /dev/null +++ b/packages/js-core/test/replaceMockable.ts @@ -0,0 +1,19 @@ +import { mockableReplacements } from '../src/util/mockable' +import { registerCleanupTask } from './registerCleanupTask' + +/** + * Registers a mock replacement for a mockable value during a test. + * Automatically cleaned up after each test via registerCleanupTask. + * + * @param value - The original value (must be the same reference passed to mockable()). + * @param replacement - The mock replacement. + */ +export function replaceMockable(value: T, replacement: T): void { + if (mockableReplacements.has(value)) { + throw new Error('Mock has already been set') + } + mockableReplacements.set(value, replacement) + registerCleanupTask(() => { + mockableReplacements.delete(value) + }) +}