Skip to content

Commit 53caf2b

Browse files
author
naman-contentstack
committed
fix: enhance error logging in ImportAssetTypes and ImportAssets
1 parent eaa18a6 commit 53caf2b

16 files changed

Lines changed: 349 additions & 167 deletions

File tree

‎.talismanrc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,8 @@ fileignoreconfig:
1919
checksum: 7b043a59fc9c523d5f772c1b81d6d4b6c65fb7f8edb8df73e48ba821e7298f0b
2020
- filename: packages/contentstack-content-type/eslint.config.js
2121
checksum: 26da78717a38d8e7464a069626213dd3010efa6e50f91efbc996f26b18346948
22+
- filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts
23+
checksum: 22708ea1e27a48a5741426a8e17e5d8b243864d877066861bc275d82393002eb
24+
- filename: packages/contentstack-asset-management/src/export/assets.ts
25+
checksum: b169481a31393a9036fbe4d41429bfee3d0f321629f01a72089469ddf5e8826d
2226
version: '1.0'

‎packages/contentstack-asset-management/src/constants/index.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,13 @@ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [
1313
'created_by',
1414
'updated_at',
1515
'updated_by',
16-
'is_system',
1716
'asset_types_count',
1817
] as const;
1918
export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [
2019
'created_at',
2120
'created_by',
2221
'updated_at',
2322
'updated_by',
24-
'is_system',
2523
'category',
2624
'preview_image_url',
2725
'category_detail',

‎packages/contentstack-asset-management/src/export/asset-types.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,26 +13,29 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter {
1313
super(apiConfig, exportContext);
1414
}
1515

16-
async start(spaceUid: string): Promise<void> {
16+
async start(spaceUid: string): Promise<number> {
1717
await this.init();
1818

1919
log.debug('Starting shared asset types export process...', this.exportContext.context);
20+
log.info('Exporting shared asset types...', this.exportContext.context);
2021

2122
const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid, this.apiPageSize, this.apiFetchConcurrency);
2223
const items = getArrayFromResponse(assetTypesData, 'asset_types');
2324
const dir = this.getAssetTypesDir();
24-
if (items.length === 0) {
25-
log.info('No asset types to export, writing empty asset-types', this.exportContext.context);
26-
} else {
27-
log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
28-
}
2925
await this.writeItemsToChunkedJson(
3026
dir,
3127
'asset-types.json',
3228
'asset_types',
3329
['uid', 'title', 'category', 'file_extension'],
3430
items,
3531
);
32+
log.info(
33+
items.length === 0
34+
? 'No asset types to export'
35+
: `Exported ${items.length} shared asset type(s)`,
36+
this.exportContext.context,
37+
);
3638
this.tick(true, `asset_types (${items.length})`, null);
39+
return items.length;
3740
}
3841
}

‎packages/contentstack-asset-management/src/export/assets.ts‎

Lines changed: 55 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,27 @@ import { configHandler, log, FsUtility } from '@contentstack/cli-utilities';
77
import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
88
import type { ExportContext } from '../types/export-types';
99
import { CSAssetsExportAdapter } from './base';
10-
import { writeStreamToFile } from '../utils/export-helpers';
10+
import { writeStreamToFile, getArrayFromResponse } from '../utils/export-helpers';
1111
import { forEachChunkedJsonStore } from '../utils/chunked-json-reader';
1212
import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry';
1313
import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter';
1414
import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
1515

16-
const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid'];
16+
// `locale` is part of the storage key so multi-locale variants of the same uid are kept
17+
// as distinct records (each locale has its own binary) instead of collapsing to one.
18+
const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid', 'locale'];
1719

18-
type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string };
20+
type AssetRecord = {
21+
uid?: string;
22+
_uid?: string;
23+
url?: string;
24+
filename?: string;
25+
file_name?: string;
26+
locale?: string;
27+
};
28+
29+
/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */
30+
export type SpaceExportCounts = { assets: number; folders: number };
1931

2032
export default class ExportAssets extends CSAssetsExportAdapter {
2133
constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
@@ -26,7 +38,7 @@ export default class ExportAssets extends CSAssetsExportAdapter {
2638
return Boolean(asset?.url && (asset?.uid ?? asset?._uid));
2739
}
2840

29-
async start(workspace: LinkedWorkspace, spaceDir: string): Promise<void> {
41+
async start(workspace: LinkedWorkspace, spaceDir: string): Promise<SpaceExportCounts> {
3042
await this.init();
3143

3244
log.debug(`Starting assets export for space ${workspace.space_uid}`, this.exportContext.context);
@@ -44,7 +56,9 @@ export default class ExportAssets extends CSAssetsExportAdapter {
4456
const onPage = (items: unknown[]) => {
4557
if (items.length === 0) return;
4658
if (!fsWriter) fsWriter = this.createChunkedJsonWriter(assetsDir, 'assets.json', 'assets', ASSET_META_KEYS);
47-
fsWriter.writeIntoFile(items as Record<string, string>[], { mapKeyVal: true });
59+
// Composite key (uid + locale) keeps each localized variant — a plain uid key would let the
60+
// last locale overwrite the rest, silently dropping their binaries.
61+
fsWriter.writeIntoFile(items as Record<string, string>[], { mapKeyVal: true, keyName: ['uid', 'locale'] });
4862
totalStreamed += items.length;
4963
for (const asset of items as AssetRecord[]) if (this.isDownloadable(asset)) downloadableCount += 1;
5064
};
@@ -75,14 +89,17 @@ export default class ExportAssets extends CSAssetsExportAdapter {
7589
this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null);
7690

7791
log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
78-
await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
92+
const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
93+
94+
const folderCount = getArrayFromResponse(folders, 'folders').length;
95+
return { assets: assetsDownloaded, folders: folderCount };
7996
}
8097

8198
/**
8299
* Download asset binaries by reading the just-written chunked `assets.json` back from disk
83100
* (one chunk at a time), so we never re-materialize the whole asset list in memory.
84101
*/
85-
private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<void> {
102+
private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<number> {
86103
const filesDir = pResolve(assetsDir, 'files');
87104
await mkdir(filesDir, { recursive: true });
88105

@@ -105,6 +122,27 @@ export default class ExportAssets extends CSAssetsExportAdapter {
105122
chunkReadLogLabel: 'assets',
106123
onOpenError: (err) => log.debug(`Could not open assets.json for download: ${err}`, this.exportContext.context),
107124
onEmptyIndexer: () => log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context),
125+
// A chunk that fails to read back would otherwise drop its downloads silently. `records` are
126+
// recovered from metadata.json, so we count + surface each lost asset by identity here — no
127+
// separate full-metadata reconcile (which would re-materialize the whole set every run).
128+
onChunkError: (records, err) => {
129+
log.error(
130+
`Failed to read an asset chunk back from disk during download for space ${spaceUid}: ${
131+
(err as Error)?.message ?? String(err)
132+
}`,
133+
this.exportContext.context,
134+
);
135+
for (const rec of records) {
136+
if (!this.isDownloadable(rec)) continue;
137+
downloadFail += 1;
138+
const label = rec.file_name ?? rec.filename ?? rec.uid ?? 'asset';
139+
this.tick(false, `asset: ${label}`, 'Asset chunk unreadable');
140+
log.error(
141+
`Asset ${rec.uid ?? '<unknown>'} (locale ${rec.locale ?? 'n/a'}) not downloaded — chunk unreadable for space ${spaceUid}`,
142+
this.exportContext.context,
143+
);
144+
}
145+
},
108146
},
109147
async (records) => {
110148
const valid = records.filter((asset) => this.isDownloadable(asset));
@@ -141,7 +179,8 @@ export default class ExportAssets extends CSAssetsExportAdapter {
141179
const body = response.body;
142180
if (!body) throw new Error('No response body');
143181
const nodeStream = Readable.fromWeb(body as Parameters<typeof Readable.fromWeb>[0]);
144-
const assetFolderPath = pResolve(filesDir, uid);
182+
// Locale-scoped path keeps each localized variant's binary distinct under the same uid.
183+
const assetFolderPath = asset.locale ? pResolve(filesDir, uid, asset.locale) : pResolve(filesDir, uid);
145184
await mkdir(assetFolderPath, { recursive: true });
146185
const filePath = pResolve(assetFolderPath, filename);
147186
await writeStreamToFile(nodeStream, filePath);
@@ -153,35 +192,29 @@ export default class ExportAssets extends CSAssetsExportAdapter {
153192
downloadFail += 1;
154193
const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
155194
this.tick(false, `asset: ${filename}`, err);
156-
log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
195+
log.error(
196+
`Failed to download asset ${uid} (${filename}): ${(e as Error)?.message ?? String(e)}`,
197+
this.exportContext.context,
198+
);
157199
}
158200
};
159201

160202
await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler);
161203
},
162204
);
163205

164-
// Completeness check: a chunk that fails to read back is skipped (logged at debug) by
165-
// forEachChunkedJsonStore, which would silently drop those downloads. Reconcile attempts
166-
// (ok + failed) against what streaming counted as downloadable.
167-
const attempted = downloadOk + downloadFail;
168-
if (attempted < expectedDownloads) {
169-
log.warn(
170-
`Asset downloads for space ${spaceUid} incomplete: expected ${expectedDownloads}, attempted ${attempted}` +
171-
` — ${expectedDownloads - attempted} asset(s) were never read back for download.`,
172-
this.exportContext.context,
173-
);
174-
}
175-
176206
log.info(
177207
downloadFail === 0
178208
? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
179209
: `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`,
180210
this.exportContext.context,
181211
);
182212
log.debug(
183-
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`,
213+
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, expected=${expectedDownloads}`,
184214
this.exportContext.context,
185215
);
216+
217+
return downloadOk;
186218
}
219+
187220
}

‎packages/contentstack-asset-management/src/export/fields.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,21 @@ export default class ExportFields extends CSAssetsExportAdapter {
1313
super(apiConfig, exportContext);
1414
}
1515

16-
async start(spaceUid: string): Promise<void> {
16+
async start(spaceUid: string): Promise<number> {
1717
await this.init();
1818

1919
log.debug('Starting shared fields export process...', this.exportContext.context);
20+
log.info('Exporting shared fields...', this.exportContext.context);
2021

2122
const fieldsData = await this.getWorkspaceFields(spaceUid, this.apiPageSize, this.apiFetchConcurrency);
2223
const items = getArrayFromResponse(fieldsData, 'fields');
2324
const dir = this.getFieldsDir();
24-
if (items.length === 0) {
25-
log.info('No field items to export, writing empty fields', this.exportContext.context);
26-
} else {
27-
log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
28-
}
2925
await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
26+
log.info(
27+
items.length === 0 ? 'No fields to export' : `Exported ${items.length} shared field(s)`,
28+
this.exportContext.context,
29+
);
3030
this.tick(true, `fields (${items.length})`, null);
31+
return items.length;
3132
}
3233
}

‎packages/contentstack-asset-management/src/export/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export { ExportSpaces, exportSpaceStructure } from './spaces';
2+
export type { AssetExportCounts } from './spaces';
23
export { default as ExportAssetTypes } from './asset-types';
34
export { default as ExportFields } from './fields';
45
export { default as ExportAssets } from './assets';

‎packages/contentstack-asset-management/src/export/spaces.ts‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,17 @@ import ExportAssetTypes from './asset-types';
99
import ExportFields from './fields';
1010
import ExportWorkspace from './workspaces';
1111

12+
/**
13+
* Real entity counts for the export summary (Bug 3 — "everything under ASSETS"):
14+
* assets = downloaded binaries, folders = folder entities, plus shared asset_types and fields.
15+
*/
16+
export type AssetExportCounts = {
17+
assets: number;
18+
folders: number;
19+
assetTypes: number;
20+
fields: number;
21+
};
22+
1223
/**
1324
* Orchestrates the full Contentstack Assets export: shared asset types and fields,
1425
* then per-workspace metadata and assets (including internal download).
@@ -27,7 +38,7 @@ export class ExportSpaces {
2738
this.parentProgressManager = parent;
2839
}
2940

30-
async start(): Promise<void> {
41+
async start(): Promise<AssetExportCounts> {
3142
const {
3243
linkedWorkspaces,
3344
exportDir,
@@ -42,7 +53,7 @@ export class ExportSpaces {
4253

4354
if (!linkedWorkspaces.length) {
4455
log.debug('No linked workspaces to export', context);
45-
return;
56+
return { assets: 0, folders: 0, assetTypes: 0, fields: 0 };
4657
}
4758

4859
log.debug('Starting Contentstack Assets export process...', context);
@@ -91,6 +102,11 @@ export class ExportSpaces {
91102
const firstSpaceUid = linkedWorkspaces[0].space_uid;
92103
let bootstrapFailed = false;
93104
let anySpaceFailed = false;
105+
// Real entity counts accumulated for the summary (Bug 3).
106+
let assetsTotal = 0;
107+
let foldersTotal = 0;
108+
let assetTypesCount = 0;
109+
let fieldsCount = 0;
94110
try {
95111
progress.startProcess(PROCESS_NAMES.AM_FIELDS);
96112
progress.startProcess(PROCESS_NAMES.AM_ASSET_TYPES);
@@ -100,7 +116,10 @@ export class ExportSpaces {
100116
const exportFields = new ExportFields(apiConfig, exportContext);
101117
exportFields.setParentProgressManager(progress);
102118
try {
103-
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
119+
[assetTypesCount, fieldsCount] = await Promise.all([
120+
exportAssetTypes.start(firstSpaceUid),
121+
exportFields.start(firstSpaceUid),
122+
]);
104123
progress.completeProcess(PROCESS_NAMES.AM_FIELDS, true);
105124
progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, true);
106125
} catch (bootstrapErr) {
@@ -118,7 +137,9 @@ export class ExportSpaces {
118137
try {
119138
const exportWorkspace = new ExportWorkspace(apiConfig, exportContext);
120139
exportWorkspace.setParentProgressManager(progress);
121-
await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
140+
const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
141+
assetsTotal += spaceCounts.assets;
142+
foldersTotal += spaceCounts.folders;
122143
progress.completeProcess(spaceProcess, true);
123144
log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
124145
} catch (err) {
@@ -142,6 +163,8 @@ export class ExportSpaces {
142163
context,
143164
);
144165
log.debug('Contentstack Assets export completed', context);
166+
167+
return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount };
145168
} catch (err) {
146169
if (!bootstrapFailed) {
147170
// Mark any spaces that hadn't been processed as failed so the multibar
@@ -170,6 +193,6 @@ export class ExportSpaces {
170193
/**
171194
* Entry point for callers that prefer a function. Delegates to ExportSpaces.
172195
*/
173-
export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise<void> {
174-
await new ExportSpaces(options).start();
196+
export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise<AssetExportCounts> {
197+
return new ExportSpaces(options).start();
175198
}

‎packages/contentstack-asset-management/src/export/workspaces.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { log } from '@contentstack/cli-utilities';
55
import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
66
import type { ExportContext } from '../types/export-types';
77
import { CSAssetsExportAdapter } from './base';
8-
import ExportAssets from './assets';
8+
import ExportAssets, { type SpaceExportCounts } from './assets';
99

1010
export default class ExportWorkspace extends CSAssetsExportAdapter {
1111
constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
@@ -26,7 +26,7 @@ export default class ExportWorkspace extends CSAssetsExportAdapter {
2626
spaceDir: string,
2727
branchName: string,
2828
spaceProcessName?: string,
29-
): Promise<void> {
29+
): Promise<SpaceExportCounts> {
3030
await this.init();
3131

3232
if (spaceProcessName) {
@@ -59,7 +59,8 @@ export default class ExportWorkspace extends CSAssetsExportAdapter {
5959
if (spaceProcessName) {
6060
assetsExporter.setProcessName(spaceProcessName);
6161
}
62-
await assetsExporter.start(workspace, spaceDir);
62+
const counts = await assetsExporter.start(workspace, spaceDir);
6363
log.debug(`Exported workspace structure for space ${workspace.space_uid}`, this.exportContext.context);
64+
return counts;
6465
}
6566
}

‎packages/contentstack-asset-management/src/import/asset-types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ export default class ImportAssetTypes extends CSAssetsImportAdapter {
138138
log.debug(`Imported asset type: ${uid}`, this.importContext.context);
139139
} catch (e) {
140140
this.failureCount += 1;
141-
log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context);
141+
log.error(`Failed to import asset type ${uid}: ${(e as Error)?.message ?? String(e)}`, this.importContext.context);
142142
}
143143
});
144144
}

0 commit comments

Comments
 (0)