diff --git a/.talismanrc b/.talismanrc index 8b5afad01..93c43278d 100644 --- a/.talismanrc +++ b/.talismanrc @@ -7,4 +7,6 @@ fileignoreconfig: checksum: 2df43be75a96e1a3a00dd256628e191909aa9d7f192b672b56cb3772e074958c - filename: pnpm-lock.yaml checksum: eae269c31146f807a80978acb68b83b761742b6d8e82116b6af48dcc0156f085 +- filename: packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts + checksum: f3620fb4441b7ced00ea85831d21f6c6b8e446a0cfe36b5bfcaaaf70788b09cc version: "" diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 662da831c..9e89b3788 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -17,8 +17,11 @@ const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid']; type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }; -/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */ -export type SpaceExportCounts = { assets: number; folders: number }; +/** + * Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities; + * failedAssets = metadata records missing after permanent page failures + binary download failures). + */ +export type SpaceExportCounts = { assets: number; folders: number; failedAssets: number }; export default class ExportAssets extends CSAssetsExportAdapter { constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) { @@ -53,11 +56,22 @@ export default class ExportAssets extends CSAssetsExportAdapter { }; log.debug(`Fetching folders and streaming assets for space ${workspace.space_uid}`, this.exportContext.context); - const [folders] = await Promise.all([ + const [folders, streamResult] = await Promise.all([ this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency), this.streamWorkspaceAssets(workspace.space_uid, workspace.uid, onPage, this.apiPageSize, this.apiFetchConcurrency), ]); + // Permanently failed metadata pages (or mid-export drift) — these records never reached + // assets.json, so the download loop can't see them. Count them as failures in the summary; + // they are recoverable via a re-export or a targeted query-export, so don't abort the run. + if (streamResult.missing > 0) { + log.error( + `${streamResult.missing} asset metadata record(s) could not be fetched for space ${workspace.space_uid} — ` + + 'they are missing from this export. Re-export (or use query-export) to recover them.', + this.exportContext.context, + ); + } + if (fsWriter) fsWriter.completeFile(true); else await this.writeEmptyChunkedJson(assetsDir, 'assets.json'); log.debug(`Wrote chunked assets metadata (${totalStreamed} item(s)) under ${assetsDir}`, this.exportContext.context); @@ -78,17 +92,21 @@ export default class ExportAssets extends CSAssetsExportAdapter { this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null); log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context); - const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount); + const downloads = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount); const folderCount = getArrayFromResponse(folders, 'folders').length; - return { assets: assetsDownloaded, folders: folderCount }; + return { assets: downloads.ok, folders: folderCount, failedAssets: streamResult.missing + downloads.fail }; } /** * Download asset binaries by reading the just-written chunked `assets.json` back from disk * (one chunk at a time), so we never re-materialize the whole asset list in memory. */ - private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise { + private async downloadWorkspaceAssets( + assetsDir: string, + spaceUid: string, + expectedDownloads: number, + ): Promise<{ ok: number; fail: number }> { const filesDir = pResolve(assetsDir, 'files'); await mkdir(filesDir, { recursive: true }); @@ -202,7 +220,7 @@ export default class ExportAssets extends CSAssetsExportAdapter { this.exportContext.context, ); - return downloadOk; + return { ok: downloadOk, fail: downloadFail }; } } diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index c41070158..df1bc6f57 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -18,6 +18,8 @@ export type AssetExportCounts = { folders: number; assetTypes: number; fields: number; + /** Assets missing from the export: permanently failed metadata pages + failed binary downloads. */ + failedAssets: number; }; /** @@ -53,7 +55,7 @@ export class ExportSpaces { if (!linkedWorkspaces.length) { log.debug('No linked workspaces to export', context); - return { assets: 0, folders: 0, assetTypes: 0, fields: 0 }; + return { assets: 0, folders: 0, assetTypes: 0, fields: 0, failedAssets: 0 }; } log.debug('Starting Contentstack Assets export process...', context); @@ -105,6 +107,7 @@ export class ExportSpaces { // Real entity counts accumulated for the summary (Bug 3). let assetsTotal = 0; let foldersTotal = 0; + let failedAssetsTotal = 0; let assetTypesCount = 0; let fieldsCount = 0; try { @@ -140,6 +143,7 @@ export class ExportSpaces { const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess); assetsTotal += spaceCounts.assets; foldersTotal += spaceCounts.folders; + failedAssetsTotal += spaceCounts.failedAssets; progress.completeProcess(spaceProcess, true); log.debug(`Exported workspace structure for space ${ws.space_uid}`, context); } catch (err) { @@ -157,14 +161,20 @@ export class ExportSpaces { } log.info( - anySpaceFailed + anySpaceFailed || failedAssetsTotal > 0 ? 'Contentstack Assets export completed with errors in one or more spaces' : 'Contentstack Assets export completed successfully', context, ); log.debug('Contentstack Assets export completed', context); - return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount }; + return { + assets: assetsTotal, + folders: foldersTotal, + assetTypes: assetTypesCount, + fields: fieldsCount, + failedAssets: failedAssetsTotal, + }; } catch (err) { if (!bootstrapFailed) { // Mark any spaces that hadn't been processed as failed so the multibar diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 56e45bfff..7085e9858 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -159,6 +159,21 @@ export type SearchAssetsParams = { limit?: number; }; +/** Response shape of GET /api/bff/spaces/{space_uid}/assets/count. */ +export type AssetCountResponse = { + count: number; +}; + +/** + * Result of streaming a workspace's assets. `missing` is the gap between the authoritative + * count-API total and the records actually streamed (permanently failed pages and/or items + * changed mid-export) — surfaced as failures in the export summary rather than aborting the run. + */ +export type StreamWorkspaceAssetsResult = { + streamed: number; + missing: number; +}; + /** Response shape from POST /api/search for assets. */ export type SearchAssetsResponse = { count?: number; @@ -174,14 +189,15 @@ export interface ICSAssetsAdapter { listSpaces(pageSize?: number, fetchConcurrency?: number): Promise; getSpace(spaceUid: string): Promise; getWorkspaceFields(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise; - getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; + getAssetsCount(spaceUid: string, workspaceUid?: string): Promise; + getFoldersCount(spaceUid: string, workspaceUid?: string): Promise; streamWorkspaceAssets( spaceUid: string, workspaceUid: string | undefined, onPage: (items: unknown[]) => void | Promise, pageSize?: number, fetchConcurrency?: number, - ): Promise; + ): Promise; getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; getWorkspaceAssetTypes(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise; searchAssets(params: SearchAssetsParams): Promise; diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index 28b0e7a11..517bcf551 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -8,6 +8,7 @@ import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '.. import type { CSAssetsAPIConfig, + AssetCountResponse, AssetTypesResponse, BulkDeleteAssetsPayload, BulkDeleteAssetsResponse, @@ -23,6 +24,7 @@ import type { SearchAssetsParams, SearchAssetsResponse, Space, + StreamWorkspaceAssetsResult, SpaceResponse, SpacesListResponse, } from '../types/cs-assets-api'; @@ -243,12 +245,15 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { async listSpaces(pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { log.debug('Fetching all spaces in org', this.config.context); + const total = await this.getListTotal('', '/api/spaces'); const items = await this.fetchAllPages( '', '/api/spaces', 'spaces', pageSize, fetchConcurrency, + {}, + total, ); log.debug(`Fetched ${items.length} space(s)`, this.config.context); return { spaces: items as Space[], count: items.length }; @@ -271,33 +276,75 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, ): Promise { log.debug(`Fetching fields for space: ${spaceUid}`, this.config.context); - const items = await this.fetchAllPages(spaceUid, '/api/fields', 'fields', pageSize, fetchConcurrency, {}); + const total = await this.getListTotal(spaceUid, '/api/fields'); + const items = await this.fetchAllPages(spaceUid, '/api/fields', 'fields', pageSize, fetchConcurrency, {}, total); log.debug(`Fetched fields (count: ${items.length})`, this.config.context); return { fields: items, count: items.length } as FieldsResponse; } /** - * GET a workspace collection (assets or folders), log count, and return result. + * GET /api/bff/spaces/{spaceUid}/assets/count — exact total for a space's assets + * (`isDir: false`) or folders (`isDir: true`). Unlike the `count` field on the paginated + * collection responses (capped at 10k by the server), this endpoint returns the true total, + * so it is the authoritative source for pagination planning. Throws when the endpoint is + * unavailable or the response has no numeric count — paginating without the true total + * would silently truncate data. A `count` of 0 is valid (genuinely empty space). */ - private async getWorkspaceCollection( + private async getEntityCount(spaceUid: string, workspaceUid: string | undefined, isDir: boolean): Promise { + const path = `/api/bff/spaces/${encodeURIComponent(spaceUid)}/assets/count`; + const queryParams: Record = { is_dir: String(isDir) }; + if (workspaceUid) queryParams.workspace = workspaceUid; + const result = await this.getSpaceLevel(spaceUid, path, queryParams); + const count = Number(result?.count); + if (!Number.isFinite(count)) { + throw new Error(`CS Assets API returned no numeric count for ${path} (space ${spaceUid})`); + } + log.debug(`Fetched ${isDir ? 'folder' : 'asset'} count for space ${spaceUid}: ${count}`, this.config.context); + return count; + } + + /** Exact asset count for a space (see {@link getEntityCount}). */ + async getAssetsCount(spaceUid: string, workspaceUid?: string): Promise { + return this.getEntityCount(spaceUid, workspaceUid, false); + } + + /** Exact folder count for a space — same count endpoint with `is_dir=true` (see {@link getEntityCount}). */ + async getFoldersCount(spaceUid: string, workspaceUid?: string): Promise { + return this.getEntityCount(spaceUid, workspaceUid, true); + } + + /** + * Resolve the total item count for a list endpoint that has NO dedicated count API + * (spaces, fields, asset types) by probing it with `limit=1` and reading the reported `count`. + * Assets and folders must NOT use this — their list `count` is capped at 10k by the server; + * use {@link getAssetsCount} instead. + */ + private async getListTotal( spaceUid: string, path: string, - logLabel: string, - queryParams: Record = {}, - ): Promise { - log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, path, queryParams); - const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); - log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); - return result; + baseParams: Record = {}, + ): Promise { + const probe = await this.getSpaceLevel>(spaceUid, path, { + ...baseParams, limit: '1', skip: '0', + }); + const count = Number(probe?.count); + if (!Number.isFinite(count)) { + // `count: 0` is a valid empty list; a MISSING count is a malformed response — failing loudly + // beats silently planning zero pages and "succeeding" with an empty export. + throw new Error(`CS Assets API returned no numeric count for ${path} (limit=1 probe)`); + } + return count; } - /** - * Core pagination: read the total `count` from page 0, then drive the remaining pages through - * {@link makeConcurrentCall}. Every page (including page 0) is handed to `onPage` — writes are - * serialized through a promise chain so a streaming sink (e.g. FsUtility) is never called - * reentrantly while pages fetch concurrently. Returns the number of items seen. + * Core pagination: given the authoritative `total` (resolved by the caller — count API for + * assets/folders, `limit=1` probe for other lists), drive every page through + * {@link makeConcurrentCall}. Each page is handed to `onPage` — writes are serialized through + * a promise chain so a streaming sink (e.g. FsUtility) is never called reentrantly while pages + * fetch concurrently. Returns the number of items seen. + * + * `paginate` never derives the total itself: a single flow driven by a passed-in total means + * there is no code path where a server-capped response `count` can silently truncate the plan. * * Peak memory is bounded by the sink: the array wrapper holds everything, but a disk-writing * sink keeps only the in-flight pages (~concurrency × pageSize). @@ -310,15 +357,10 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { concurrency: number, baseParams: Record, onPage: (items: unknown[]) => void | Promise, - ): Promise { - const first = await this.getSpaceLevel>(spaceUid, path, { - ...baseParams, limit: String(pageSize), skip: '0', - }); - - const total: number = Number(first?.count ?? 0); - const firstItems: unknown[] = Array.isArray(first?.[itemsKey]) ? (first[itemsKey] as unknown[]) : []; - + total: number, + ): Promise<{ collected: number; failedPages: number }> { let collected = 0; + let failedPages = 0; let writeFailures = 0; let writeChain: Promise = Promise.resolve(); const enqueue = (items: unknown[]) => { @@ -334,17 +376,14 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { }); }; - enqueue(firstItems); - - if (firstItems.length < total) { - // Remaining skip offsets (page 0 already fetched), pre-chunked into batches of `concurrency`. + if (total > 0) { + // All skip offsets derived from the authoritative total, pre-chunked into batches of `concurrency`. const skips: string[] = Array.from( - { length: Math.ceil(total / pageSize) - 1 }, - (_, i) => String((i + 1) * pageSize), + { length: Math.ceil(total / pageSize) }, + (_, i) => String(i * pageSize), ); const apiBatches = chunk(skips, concurrency); - let failedPages = 0; const onSuccess = ({ response }: any) => { const items = Array.isArray(response?.[itemsKey]) ? (response[itemsKey] as unknown[]) : []; enqueue(items); @@ -368,7 +407,7 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { }); // Completeness check: the export "succeeding" with silently-missing pages is the worst failure - // mode for a backup/migration, so reconcile what we saw against the server's reported total. + // mode for a backup/migration, so reconcile what we saw against the authoritative total. if (collected !== total) { log.warn( `Incomplete pagination for ${itemsKey} (${path}): expected ${total}, collected ${collected}` + @@ -385,7 +424,7 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { this.config.context, ); } - return collected; + return { collected, failedPages }; } /** @@ -399,18 +438,22 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { itemsKey: string, pageSize: number, concurrency: number, - baseParams: Record = {}, + baseParams: Record, + total: number, ): Promise { const out: unknown[] = []; await this.paginate(spaceUid, path, itemsKey, pageSize, concurrency, baseParams, (items) => { out.push(...items); - }); + }, total); return out; } /** * Stream a workspace's assets page-by-page to `onPage` (e.g. an incremental chunked-JSON writer) - * instead of buffering the whole set. Returns the number of asset records streamed. + * instead of buffering the whole set. Returns the number of records streamed plus the number + * missing versus the authoritative total (`missing > 0` means page requests failed permanently + * or items changed mid-export) — callers surface `missing` as failures in the export summary; + * page-level failures are recoverable by re-exporting, so they don't abort the run. */ async streamWorkspaceAssets( spaceUid: string, @@ -418,9 +461,12 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { onPage: (items: unknown[]) => void | Promise, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, - ): Promise { + ): Promise { const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; - return this.paginate( + // The assets list response caps `count` at 10k — the dedicated count API is the only + // trustworthy total. Its failure propagates: exporting with a wrong total means silent data loss. + const total = await this.getAssetsCount(spaceUid, workspaceUid); + const { collected } = await this.paginate( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', @@ -428,7 +474,9 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { fetchConcurrency, baseParams, onPage, + total, ); + return { streamed: collected, missing: Math.max(0, total - collected) }; } /** @@ -487,21 +535,9 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } } - async getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { - const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; - const items = await this.fetchAllPages( - spaceUid, - `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, - 'assets', - pageSize, - fetchConcurrency, - baseParams, - ); - return { assets: items, count: items.length }; - } - async getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const total = await this.getFoldersCount(spaceUid, workspaceUid); const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, @@ -509,6 +545,7 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { pageSize, fetchConcurrency, baseParams, + total, ); return { folders: items, count: items.length }; } @@ -519,9 +556,9 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, ): Promise { log.debug(`Fetching asset types for space: ${spaceUid}`, this.config.context); - const items = await this.fetchAllPages(spaceUid, '/api/asset_types', 'asset_types', pageSize, fetchConcurrency, { - include_fields: 'true', - }); + const baseParams = { include_fields: 'true' }; + const total = await this.getListTotal(spaceUid, '/api/asset_types', baseParams); + const items = await this.fetchAllPages(spaceUid, '/api/asset_types', 'asset_types', pageSize, fetchConcurrency, baseParams, total); log.debug(`Fetched asset types (count: ${items.length})`, this.config.context); return { asset_types: items, count: items.length } as AssetTypesResponse; } diff --git a/packages/contentstack-asset-management/test/unit/export/assets.test.ts b/packages/contentstack-asset-management/test/unit/export/assets.test.ts index ae930da8b..31eb71c8d 100644 --- a/packages/contentstack-asset-management/test/unit/export/assets.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/assets.test.ts @@ -42,13 +42,13 @@ describe('ExportAssets', () => { * through the `onPage` sink, and the download read-back (`forEachChunkedJsonStore`) yields the * same `items` back as one chunk (or signals empty). */ - const wireStreaming = (items: Array>) => { + const wireStreaming = (items: Array>, missing = 0) => { sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData as any); sinon .stub(ExportAssets.prototype, 'streamWorkspaceAssets') .callsFake(async (_s: string, _ws: string | undefined, onPage: (i: unknown[]) => void | Promise) => { await onPage(items); - return items.length; + return { streamed: items.length, missing }; }); sinon .stub(chunkedJsonReader, 'forEachChunkedJsonStore') @@ -151,6 +151,27 @@ describe('ExportAssets', () => { } }); + it('should count missing metadata records (failed pages) as failedAssets without aborting', async () => { + wireStreaming(assetItems, 3); // stream succeeded for 2 items, 3 records lost to failed pages + fetchStub.callsFake(async () => makeFetchResponse() as any); + + const exporter = new ExportAssets(apiConfig, exportContext); + const counts = await exporter.start(workspace, spaceDir); + + expect(counts.failedAssets).to.equal(3); + }); + + it('should count failed binary downloads in failedAssets', async () => { + wireStreaming(assetItems); + fetchStub.rejects(new Error('network failure')); + + const exporter = new ExportAssets(apiConfig, exportContext); + const counts = await exporter.start(workspace, spaceDir); + + expect(counts.assets).to.equal(0); + expect(counts.failedAssets).to.equal(assetItems.length); + }); + it('should tick per asset with success=true and null error on successful downloads', async () => { wireStreaming(assetItems); fetchStub.callsFake(async () => makeFetchResponse() as any); diff --git a/packages/contentstack-asset-management/test/unit/export/spaces.test.ts b/packages/contentstack-asset-management/test/unit/export/spaces.test.ts index 6ebece314..6c6cdd024 100644 --- a/packages/contentstack-asset-management/test/unit/export/spaces.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/spaces.test.ts @@ -39,7 +39,7 @@ describe('ExportSpaces', () => { sinon.stub(ExportAssetTypes.prototype, 'setParentProgressManager'); sinon.stub(ExportFields.prototype, 'start').resolves(); sinon.stub(ExportFields.prototype, 'setParentProgressManager'); - sinon.stub(ExportWorkspace.prototype, 'start').resolves({ assets: 0, folders: 0 }); + sinon.stub(ExportWorkspace.prototype, 'start').resolves({ assets: 0, folders: 0, failedAssets: 0 }); sinon.stub(ExportWorkspace.prototype, 'setParentProgressManager'); fakeProgress.addProcess.resetHistory(); @@ -133,7 +133,7 @@ describe('ExportSpaces', () => { it('should mark only the failing space row as failed and continue with remaining spaces', async () => { const wsStub = ExportWorkspace.prototype.start as sinon.SinonStub; wsStub.onFirstCall().rejects(new Error('workspace-error')); - wsStub.onSecondCall().resolves({ assets: 0, folders: 0 }); + wsStub.onSecondCall().resolves({ assets: 0, folders: 0, failedAssets: 0 }); const exporter = new ExportSpaces(baseOptions); // Per the plan, per-space failures must NOT abort the orchestrator — diff --git a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts index d9b86a60a..9b0fa50c1 100644 --- a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts +++ b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts @@ -153,44 +153,134 @@ describe('CSAssetsAdapter', () => { }); it('should fetch all fields across multiple pages', async () => { - getStub.onCall(0).resolves({ status: 200, data: { fields: [{ uid: 'f1' }, { uid: 'f2' }], count: 3 } }); - getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f3' }], count: 3 } }); + // Call 0 is the limit=1 probe that resolves the total; pages follow. + getStub.onCall(0).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f1' }, { uid: 'f2' }], count: 3 } }); + getStub.onCall(2).resolves({ status: 200, data: { fields: [{ uid: 'f3' }], count: 3 } }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.getWorkspaceFields('sp-1', 2, 5); - expect(getStub.callCount).to.equal(2); - expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(getStub.callCount).to.equal(3); + expect(getStub.firstCall.args[0]).to.include('limit=1'); + expect(getStub.thirdCall.args[0]).to.include('skip=2'); expect(result.count).to.equal(3); expect(result.fields).to.have.lengthOf(3); }); }); - describe('getWorkspaceAssets', () => { - it('should GET /api/spaces/{spaceUid}/assets', async () => { - getStub.resolves({ status: 200, data: { items: [] } }); + describe('getAssetsCount', () => { + it('should GET /api/bff/spaces/{spaceUid}/assets/count with is_dir=false and workspace param, and return the count', async () => { + getStub.resolves({ status: 200, data: { count: 42 } }); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.getWorkspaceAssets('sp-1'); + const result = await adapter.getAssetsCount('sp-1', 'ws-main'); - expect(getStub.firstCall.args[0]).to.include('/api/spaces/sp-1/assets'); + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('/api/bff/spaces/sp-1/assets/count'); + expect(path).to.include('is_dir=false'); + expect(path).to.include('workspace=ws-main'); + expect(result).to.equal(42); + }); + + it('getFoldersCount: should pass is_dir=true for folder counts', async () => { + getStub.resolves({ status: 200, data: { count: 7 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getFoldersCount('sp-1', 'ws-main'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('/api/bff/spaces/sp-1/assets/count'); + expect(path).to.include('is_dir=true'); + expect(result).to.equal(7); + }); + + it('should omit the workspace param when workspaceUid is undefined', async () => { + getStub.resolves({ status: 200, data: { count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getAssetsCount('sp-1'); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.not.include('workspace='); }); it('should URL-encode the spaceUid in the path', async () => { - getStub.resolves({ status: 200, data: { items: [] } }); + getStub.resolves({ status: 200, data: { count: 1 } }); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.getWorkspaceAssets('sp uid/special'); + await adapter.getAssetsCount('sp uid/special'); const path = getStub.firstCall.args[0] as string; expect(path).to.include('sp%20uid%2Fspecial'); }); + + it('should return 0 (not throw) for a genuinely empty space', async () => { + getStub.resolves({ status: 200, data: { count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getAssetsCount('sp-1', 'ws-main'); + + expect(result).to.equal(0); + }); + + it('should throw when the response has no numeric count', async () => { + getStub.resolves({ status: 200, data: { notice: 'ok' } }); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.getAssetsCount('sp-1', 'ws-main'); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('count'); + } + }); + + it('should throw (not fall back) when the count endpoint returns a terminal error', async () => { + getStub.resolves({ status: 404, data: { error: 'not found' } }); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.getAssetsCount('sp-1', 'ws-main'); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API GET failed'); + } + }); + }); + + describe('streamWorkspaceAssets (URLs)', () => { + it('should GET /api/spaces/{spaceUid}/assets after resolving the count', async () => { + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 1 } }; + return { status: 200, data: { assets: [{ uid: 'a1' }], count: 1 } }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.streamWorkspaceAssets('sp-1', undefined, () => {}); + + expect(getStub.firstCall.args[0]).to.include('/api/bff/spaces/sp-1/assets/count'); + expect(getStub.secondCall.args[0]).to.include('/api/spaces/sp-1/assets'); + }); + + it('should URL-encode the spaceUid in the path', async () => { + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 1 } }; + return { status: 200, data: { assets: [{ uid: 'a1' }], count: 1 } }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.streamWorkspaceAssets('sp uid/special', undefined, () => {}); + + expect(getStub.firstCall.args[0] as string).to.include('sp%20uid%2Fspecial'); + expect(getStub.secondCall.args[0] as string).to.include('sp%20uid%2Fspecial'); + }); }); describe('getWorkspaceFolders', () => { - it('should GET /api/spaces/{spaceUid}/folders', async () => { - getStub.resolves({ status: 200, data: [] }); + it('should GET /api/spaces/{spaceUid}/folders after resolving the folder count', async () => { + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 1 } }; + return { status: 200, data: { folders: [{ uid: 'f1' }], count: 1 } }; + }); const adapter = new CSAssetsAdapter(baseConfig); await adapter.getWorkspaceFolders('sp-1'); - expect(getStub.firstCall.args[0]).to.include('/api/spaces/sp-1/folders'); + expect(getStub.firstCall.args[0]).to.include('/api/bff/spaces/sp-1/assets/count'); + expect(getStub.firstCall.args[0]).to.include('is_dir=true'); + expect(getStub.secondCall.args[0]).to.include('/api/spaces/sp-1/folders'); }); }); @@ -209,14 +299,18 @@ describe('CSAssetsAdapter', () => { }); it('should fetch all asset types across multiple pages, preserving include_fields', async () => { - getStub.onCall(0).resolves({ status: 200, data: { asset_types: [{ uid: 'at1' }, { uid: 'at2' }], count: 3 } }); - getStub.onCall(1).resolves({ status: 200, data: { asset_types: [{ uid: 'at3' }], count: 3 } }); + // Call 0 is the limit=1 probe that resolves the total; pages follow. + getStub.onCall(0).resolves({ status: 200, data: { asset_types: [{ uid: 'at1' }], count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { asset_types: [{ uid: 'at1' }, { uid: 'at2' }], count: 3 } }); + getStub.onCall(2).resolves({ status: 200, data: { asset_types: [{ uid: 'at3' }], count: 3 } }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.getWorkspaceAssetTypes('sp-1', 2, 5); - expect(getStub.callCount).to.equal(2); - expect(getStub.secondCall.args[0]).to.include('include_fields=true'); - expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(getStub.callCount).to.equal(3); + expect(getStub.firstCall.args[0]).to.include('limit=1'); + expect(getStub.firstCall.args[0]).to.include('include_fields=true'); + expect(getStub.thirdCall.args[0]).to.include('include_fields=true'); + expect(getStub.thirdCall.args[0]).to.include('skip=2'); expect(result.count).to.equal(3); expect(result.asset_types).to.have.lengthOf(3); }); @@ -252,10 +346,12 @@ describe('CSAssetsAdapter', () => { const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.listSpaces(100, 5); - expect(getStub.callCount).to.equal(1); + // Call 0 is the limit=1 probe; call 1 is the single page. + expect(getStub.callCount).to.equal(2); expect(getStub.firstCall.args[0]).to.include('/api/spaces'); - expect(getStub.firstCall.args[0]).to.include('limit=100'); - expect(getStub.firstCall.args[0]).to.include('skip=0'); + expect(getStub.firstCall.args[0]).to.include('limit=1'); + expect(getStub.secondCall.args[0]).to.include('limit=100'); + expect(getStub.secondCall.args[0]).to.include('skip=0'); expect(result.spaces).to.deep.equal(spaces); expect(result.count).to.equal(2); }); @@ -263,13 +359,14 @@ describe('CSAssetsAdapter', () => { it('should issue additional page requests when total exceeds first page', async () => { const page1 = Array.from({ length: 2 }, (_, i) => ({ uid: `sp-${i}` })); const page2 = Array.from({ length: 1 }, (_, i) => ({ uid: `sp-${i + 2}` })); - getStub.onCall(0).resolves({ status: 200, data: { spaces: page1, count: 3 } }); - getStub.onCall(1).resolves({ status: 200, data: { spaces: page2, count: 3 } }); + getStub.onCall(0).resolves({ status: 200, data: { spaces: [page1[0]], count: 3 } }); // probe + getStub.onCall(1).resolves({ status: 200, data: { spaces: page1, count: 3 } }); + getStub.onCall(2).resolves({ status: 200, data: { spaces: page2, count: 3 } }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.listSpaces(2, 5); - expect(getStub.callCount).to.equal(2); - expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(getStub.callCount).to.equal(3); + expect(getStub.thirdCall.args[0]).to.include('skip=2'); expect(result.spaces).to.have.lengthOf(3); expect(result.count).to.equal(3); }); @@ -279,45 +376,53 @@ describe('CSAssetsAdapter', () => { const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.listSpaces(); + // Probe reports 0 → no page requests at all. expect(getStub.callCount).to.equal(1); + expect(getStub.firstCall.args[0]).to.include('limit=1'); expect(result.spaces).to.deep.equal([]); expect(result.count).to.equal(0); }); it('should batch additional page requests by fetchConcurrency', async () => { - // 5 total, pageSize=1, concurrency=2 → 4 additional pages in 2 batches + // 5 total, pageSize=1, concurrency=2 → probe + 5 pages in 3 batches const pages = Array.from({ length: 5 }, (_, i) => [{ uid: `sp-${i}` }]); - getStub.onCall(0).resolves({ status: 200, data: { spaces: pages[0], count: 5 } }); - getStub.onCall(1).resolves({ status: 200, data: { spaces: pages[1], count: 5 } }); - getStub.onCall(2).resolves({ status: 200, data: { spaces: pages[2], count: 5 } }); - getStub.onCall(3).resolves({ status: 200, data: { spaces: pages[3], count: 5 } }); - getStub.onCall(4).resolves({ status: 200, data: { spaces: pages[4], count: 5 } }); + getStub.onCall(0).resolves({ status: 200, data: { spaces: pages[0], count: 5 } }); // probe + getStub.onCall(1).resolves({ status: 200, data: { spaces: pages[0], count: 5 } }); + getStub.onCall(2).resolves({ status: 200, data: { spaces: pages[1], count: 5 } }); + getStub.onCall(3).resolves({ status: 200, data: { spaces: pages[2], count: 5 } }); + getStub.onCall(4).resolves({ status: 200, data: { spaces: pages[3], count: 5 } }); + getStub.onCall(5).resolves({ status: 200, data: { spaces: pages[4], count: 5 } }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.listSpaces(1, 2); - expect(getStub.callCount).to.equal(5); + expect(getStub.callCount).to.equal(6); expect(result.spaces).to.have.lengthOf(5); }); it('should request each page with its OWN skip when pages run concurrently (no shared-mutation race)', async () => { - // total 6, pageSize 2, concurrency 5 → page0 inline, then skips [2,4] in a single concurrent batch. - // If makeAPICall did not snapshot queryParam, both concurrent calls would read the last skip. + // total 6, pageSize 2, concurrency 5 → probe, then skips [0,2,4] in a single concurrent batch. + // If makeAPICall did not snapshot queryParam, concurrent calls would read the last skip. const bySkip: Record = { '0': { spaces: [{ uid: 's0' }, { uid: 's1' }], count: 6 }, '2': { spaces: [{ uid: 's2' }, { uid: 's3' }], count: 6 }, '4': { spaces: [{ uid: 's4' }, { uid: 's5' }], count: 6 }, }; getStub.callsFake(async (path: string) => { - const skip = new URL(`https://x${path}`).searchParams.get('skip') ?? '0'; - return { status: 200, data: bySkip[skip] }; + const params = new URL(`https://x${path}`).searchParams; + if (params.get('limit') === '1') return { status: 200, data: { spaces: [{ uid: 's0' }], count: 6 } }; + return { status: 200, data: bySkip[params.get('skip') ?? '0'] }; }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.listSpaces(2, 5); const uids = (result.spaces as Array<{ uid: string }>).map((s) => s.uid).sort(); expect(uids).to.deep.equal(['s0', 's1', 's2', 's3', 's4', 's5']); - const requestedSkips = getStub.getCalls().map((c) => new URL(`https://x${c.args[0]}`).searchParams.get('skip')); - expect([...requestedSkips].sort()).to.deep.equal(['0', '2', '4']); + const pageSkips = getStub + .getCalls() + .map((c) => new URL(`https://x${c.args[0]}`).searchParams) + .filter((p) => p.get('limit') !== '1') + .map((p) => p.get('skip')); + expect([...pageSkips].sort()).to.deep.equal(['0', '2', '4']); }); }); @@ -352,23 +457,27 @@ describe('CSAssetsAdapter', () => { const retryConfig: CSAssetsAPIConfig = { ...baseConfig, retryBaseDelayMs: 0 }; it('should retry a 429 and then succeed', async () => { + // Calls 0-2: probe retried through two 429s; call 3: the single page. getStub.onCall(0).resolves({ status: 429, data: {} }); getStub.onCall(1).resolves({ status: 429, data: {} }); getStub.onCall(2).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); + getStub.onCall(3).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); const adapter = new CSAssetsAdapter(retryConfig); const result = await adapter.getWorkspaceFields('sp-1'); - expect(getStub.callCount).to.equal(3); + expect(getStub.callCount).to.equal(4); expect(result.fields).to.deep.equal([{ uid: 'f1' }]); }); it('should retry a 5xx and then succeed', async () => { + // Calls 0-1: probe retried through a 503; call 2: the single page. getStub.onCall(0).resolves({ status: 503, data: {} }); getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); + getStub.onCall(2).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); const adapter = new CSAssetsAdapter(retryConfig); const result = await adapter.getWorkspaceFields('sp-1'); - expect(getStub.callCount).to.equal(2); + expect(getStub.callCount).to.equal(3); expect(result.fields).to.deep.equal([{ uid: 'f1' }]); }); @@ -386,23 +495,41 @@ describe('CSAssetsAdapter', () => { }); }); - describe('getWorkspaceAssets (paginated)', () => { - it('should fetch all assets across multiple pages', async () => { + describe('streamWorkspaceAssets (paginated)', () => { + it('should stream all assets across multiple pages and report zero missing', async () => { const page1 = [{ uid: 'a-1' }, { uid: 'a-2' }]; const page2 = [{ uid: 'a-3' }]; - getStub.onCall(0).resolves({ status: 200, data: { assets: page1, count: 3 } }); - getStub.onCall(1).resolves({ status: 200, data: { assets: page2, count: 3 } }); + getStub.onCall(0).resolves({ status: 200, data: { count: 3 } }); // count API + getStub.onCall(1).resolves({ status: 200, data: { assets: page1, count: 3 } }); + getStub.onCall(2).resolves({ status: 200, data: { assets: page2, count: 3 } }); const adapter = new CSAssetsAdapter(baseConfig); - const result = await adapter.getWorkspaceAssets('sp-1', undefined, 2, 5) as any; + const streamed: unknown[] = []; + const result = await adapter.streamWorkspaceAssets('sp-1', undefined, (items) => { + streamed.push(...items); + }, 2, 5); - expect(result.assets).to.have.lengthOf(3); - expect(result.count).to.equal(3); + expect(streamed).to.have.lengthOf(3); + expect(result).to.deep.equal({ streamed: 3, missing: 0 }); + }); + + it('should report missing records when a page request fails permanently (export continues)', async () => { + // Count says 4; pageSize 2 → 2 pages; the skip=2 page 404s → 2 records lost but no throw. + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 4 } }; + const skip = new URL(`https://x${path}`).searchParams.get('skip'); + if (skip === '2') return { status: 404, data: { error: 'gone' } }; + return { status: 200, data: { assets: [{ uid: 'a0' }, { uid: 'a1' }], count: 4 } }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.streamWorkspaceAssets('sp-1', 'ws-main', () => {}, 2, 5); + + expect(result).to.deep.equal({ streamed: 2, missing: 2 }); }); it('should include workspace query param when workspaceUid is provided', async () => { getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.getWorkspaceAssets('sp-1', 'ws-main', 100, 5); + await adapter.streamWorkspaceAssets('sp-1', 'ws-main', () => {}, 100, 5); const path = getStub.firstCall.args[0] as string; expect(path).to.include('workspace=ws-main'); @@ -411,7 +538,7 @@ describe('CSAssetsAdapter', () => { it('should NOT include workspace param when workspaceUid is undefined', async () => { getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.getWorkspaceAssets('sp-1', undefined, 100, 5); + await adapter.streamWorkspaceAssets('sp-1', undefined, () => {}, 100, 5); const path = getStub.firstCall.args[0] as string; expect(path).to.not.include('workspace='); @@ -422,8 +549,9 @@ describe('CSAssetsAdapter', () => { it('should fetch all folders across multiple pages', async () => { const page1 = [{ uid: 'f-1' }]; const page2 = [{ uid: 'f-2' }]; - getStub.onCall(0).resolves({ status: 200, data: { folders: page1, count: 2 } }); - getStub.onCall(1).resolves({ status: 200, data: { folders: page2, count: 2 } }); + getStub.onCall(0).resolves({ status: 200, data: { count: 2 } }); // count API (is_dir=true) + getStub.onCall(1).resolves({ status: 200, data: { folders: page1, count: 2 } }); + getStub.onCall(2).resolves({ status: 200, data: { folders: page2, count: 2 } }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.getWorkspaceFolders('sp-1', undefined, 1, 5) as any; @@ -441,6 +569,129 @@ describe('CSAssetsAdapter', () => { }); }); + describe('count-driven pagination (single flow)', () => { + it('streamWorkspaceAssets: fetches the exact total from the count API first, then pages', async () => { + // Count API says 5; page responses report a (capped-style) count of 2 — the count API total must win. + const bySkip: Record = { + '0': { assets: [{ uid: 'a0' }, { uid: 'a1' }], count: 2 }, + '2': { assets: [{ uid: 'a2' }, { uid: 'a3' }], count: 2 }, + '4': { assets: [{ uid: 'a4' }], count: 2 }, + }; + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 5 } }; + const skip = new URL(`https://x${path}`).searchParams.get('skip') ?? '0'; + return { status: 200, data: bySkip[skip] }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const streamed: unknown[] = []; + const result = await adapter.streamWorkspaceAssets('sp-1', 'ws-main', (items) => { + streamed.push(...items); + }, 2, 5); + + const firstPath = getStub.firstCall.args[0] as string; + expect(firstPath).to.include('/api/bff/spaces/sp-1/assets/count'); + expect(firstPath).to.include('is_dir=false'); + expect(streamed).to.have.lengthOf(5); + expect(result).to.deep.equal({ streamed: 5, missing: 0 }); + }); + + it('getWorkspaceFolders: uses is_dir=true for the count call', async () => { + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 1 } }; + return { status: 200, data: { folders: [{ uid: 'f0' }], count: 1 } }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceFolders('sp-1', 'ws-main', 100, 5) as any; + + const firstPath = getStub.firstCall.args[0] as string; + expect(firstPath).to.include('/api/bff/spaces/sp-1/assets/count'); + expect(firstPath).to.include('is_dir=true'); + expect(result.folders).to.have.lengthOf(1); + }); + + it('streamWorkspaceAssets: fetches count first and streams every page', async () => { + const bySkip: Record = { + '0': { assets: [{ uid: 'a0' }, { uid: 'a1' }], count: 2 }, + '2': { assets: [{ uid: 'a2' }], count: 2 }, + }; + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 3 } }; + const skip = new URL(`https://x${path}`).searchParams.get('skip') ?? '0'; + return { status: 200, data: bySkip[skip] }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const streamed: unknown[] = []; + const result = await adapter.streamWorkspaceAssets('sp-1', 'ws-main', (items) => { + streamed.push(...items); + }, 2, 5); + + expect(getStub.firstCall.args[0]).to.include('/assets/count'); + expect(result.streamed).to.equal(3); + expect(streamed).to.have.lengthOf(3); + }); + + it('streamWorkspaceAssets: propagates a count API failure without fetching any pages', async () => { + getStub.resolves({ status: 404, data: { error: 'not found' } }); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.streamWorkspaceAssets('sp-1', 'ws-main', () => {}, 2, 5); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API GET failed'); + } + // Only the count request went out — pagination never started. + expect(getStub.callCount).to.equal(1); + expect(getStub.firstCall.args[0]).to.include('/assets/count'); + }); + + it('streamWorkspaceAssets: makes no page requests when the count is 0', async () => { + getStub.callsFake(async (path: string) => { + if (path.includes('/assets/count')) return { status: 200, data: { count: 0 } }; + return { status: 200, data: { assets: [], count: 0 } }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.streamWorkspaceAssets('sp-1', 'ws-main', () => {}, 100, 5); + + expect(getStub.callCount).to.equal(1); + expect(result).to.deep.equal({ streamed: 0, missing: 0 }); + }); + + it('listSpaces: throws when the probe response has no numeric count (malformed response ≠ empty list)', async () => { + // A genuinely empty org returns { count: 0 } and is fine; a response with NO count field is + // malformed and must fail loudly instead of silently exporting nothing. + getStub.resolves({ status: 200, data: { spaces: [] } }); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.listSpaces(2, 5); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('count'); + } + }); + + it('listSpaces: derives the total from a limit=1 probe of the list endpoint (no count API exists)', async () => { + const bySkip: Record = { + '0': { spaces: [{ uid: 's0' }, { uid: 's1' }], count: 3 }, + '2': { spaces: [{ uid: 's2' }], count: 3 }, + }; + getStub.callsFake(async (path: string) => { + const params = new URL(`https://x${path}`).searchParams; + if (params.get('limit') === '1') return { status: 200, data: { spaces: [{ uid: 's0' }], count: 3 } }; + return { status: 200, data: bySkip[params.get('skip') ?? '0'] }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(2, 5); + + const probePath = getStub.firstCall.args[0] as string; + expect(probePath).to.include('/api/spaces'); + expect(probePath).to.include('limit=1'); + expect(result.spaces).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + }); + describe('POST methods (createSpace, createFolder, createField, createAssetType, bulkDelete, bulkMove)', () => { let fetchStub: sinon.SinonStub; diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 6121dbd6d..5dfb193c8 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -91,7 +91,9 @@ export default class ExportAssets extends BaseClass { const assetsModule = gs.getModules().get(this.currentModuleName.toUpperCase()); if (assetsModule) { assetsModule.successCount = counts.assets; - assetsModule.failureCount = 0; + // Metadata records lost to permanently-failed page fetches + failed binary downloads — + // recoverable via re-export/query-export, so they surface as failures instead of aborting. + assetsModule.failureCount = counts.failedAssets ?? 0; } const extraRows: Array<[string, number]> = [