diff --git a/api/package-lock.json b/api/package-lock.json index 98a7df42e..fce534003 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -18422,4 +18422,4 @@ } } } -} +} \ No newline at end of file diff --git a/api/package.json b/api/package.json index 5e98ea378..45b0c02b3 100644 --- a/api/package.json +++ b/api/package.json @@ -104,4 +104,4 @@ "undici": ">=7.24.0" }, "keywords": [] -} +} \ No newline at end of file diff --git a/api/src/controllers/migration.controller.ts b/api/src/controllers/migration.controller.ts index f44d034b6..7fe65c13a 100644 --- a/api/src/controllers/migration.controller.ts +++ b/api/src/controllers/migration.controller.ts @@ -68,6 +68,11 @@ const saveMappedLocales = async (req: Request, res: Response): Promise => res.status(200).json(resp); } +const restartMigration = async (req: Request, res: Response): Promise => { + const resp = await migrationService.restartMigration(req); + res.status(200).json(resp); +} + export const migrationController = { createTestStack, deleteTestStack, @@ -76,5 +81,6 @@ export const migrationController = { getLogs, saveLocales, saveMappedLocales, - getAuditData + getAuditData, + restartMigration }; diff --git a/api/src/controllers/projects.contentMapper.controller.ts b/api/src/controllers/projects.contentMapper.controller.ts index 8b9ca120d..73c020564 100644 --- a/api/src/controllers/projects.contentMapper.controller.ts +++ b/api/src/controllers/projects.contentMapper.controller.ts @@ -164,6 +164,30 @@ const updateContentMapper = async (req: Request, res: Response): Promise = res.status(project.status).json(project); } +/** + * Retrieves entry mapping for a content type. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const getEntryMapping = async (req: Request, res: Response): Promise => { + const resp = await contentMapperService.getEntryMapping(req); + res.status(resp?.status || 200).json(resp); +}; + +/** + * Updates the status of entries. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const updateEntryStatus = async (req: Request, res: Response): Promise => { + const resp = await contentMapperService.updateEntryStatus(req); + res.status(resp?.status || 200).json(resp); +}; + export const contentMapperController = { getContentTypes, getFieldMapping, @@ -177,5 +201,7 @@ export const contentMapperController = { removeContentMapper, updateContentMapper, getExistingGlobalFields, - getSingleGlobalField + getSingleGlobalField, + getEntryMapping, + updateEntryStatus }; diff --git a/api/src/models/EntryMapper.ts b/api/src/models/EntryMapper.ts new file mode 100644 index 000000000..985dd5409 --- /dev/null +++ b/api/src/models/EntryMapper.ts @@ -0,0 +1,60 @@ +import { JSONFile } from "lowdb/node"; +import LowWithLodash from "../utils/lowdb-lodash.utils.js"; +import path from "path"; +import fs from 'node:fs'; + +/** + * Represents the advanced configuration options for a field mapper. + */ +export interface Advanced { + validationRegex: string; + mandatory: boolean; + multiple: boolean; + unique: boolean; + nonLocalizable: boolean; + embedObject: boolean; + embedObjects: any; + minChars: string; + maxChars: number; + default_value: string; + description: string; + validationErrorMessage: string; + options: any[]; +} + +/** + * Represents an entry mapper object. + */ +export interface EntryMapper { + entry_mapper: { + id: string; + projectId: string; + contentTypeId: string; + contentTypeUid: string; + entryName: string; + otherCmsEntryUid: string; + isUpdate: boolean; + contentstackEntryUid: string; + isDuplicateEntry: boolean; + }[]; +} + +const defaultData: EntryMapper = { entry_mapper: [] }; + +/** + * Creates and returns a database instance for the field mapper for a specific project. + * @param projectId - The unique identifier of the project + * @returns The database instance for the field mapper + */ +const getEntryMapperDb = (projectId: string, iteration: number) => { + fs.mkdirSync(path.join(process.cwd(), "database", projectId, iteration.toString()), { recursive: true }); + const db = new LowWithLodash( + new JSONFile( + path.join(process.cwd(), "database", projectId, iteration.toString(), 'entry-mapper.json') + ), + defaultData + ); + return db; +}; + +export default getEntryMapperDb; diff --git a/api/src/models/FieldMapper.ts b/api/src/models/FieldMapper.ts index 5be7d0528..5f994ef32 100644 --- a/api/src/models/FieldMapper.ts +++ b/api/src/models/FieldMapper.ts @@ -1,6 +1,7 @@ import { JSONFile } from "lowdb/node"; import LowWithLodash from "../utils/lowdb-lodash.utils.js"; import path from "path"; +import fs from 'node:fs'; /** * Represents the advanced configuration options for a field mapper. @@ -46,11 +47,20 @@ interface FieldMapper { const defaultData: FieldMapper = { field_mapper: [] }; /** - * Represents the database instance for the FieldMapper model. + * Creates and returns a database instance for the field mapper for a specific project. + * @param projectId - The unique identifier of the project + * @returns The database instance for the field mapper */ -const db = new LowWithLodash( - new JSONFile(path.join(process.cwd(), "database", "field-mapper.json")), - defaultData -); +const getFieldMapperDb = (projectId: string, iteration: number) => { + fs.mkdirSync(path.join(process.cwd(), "database", projectId, iteration.toString()), { recursive: true }); + const db = new LowWithLodash( + new JSONFile( + path.join(process.cwd(), "database", projectId, iteration.toString(), 'field-mapper.json') + ), + defaultData + ); + return db; +}; + +export default getFieldMapperDb; -export default db; diff --git a/api/src/models/contentTypesMapper-lowdb.ts b/api/src/models/contentTypesMapper-lowdb.ts index e4163818a..e4eedf500 100644 --- a/api/src/models/contentTypesMapper-lowdb.ts +++ b/api/src/models/contentTypesMapper-lowdb.ts @@ -1,6 +1,7 @@ import { JSONFile } from "lowdb/node"; import path from 'path'; import LowWithLodash from "../utils/lowdb-lodash.utils.js"; +import fs from 'node:fs'; /** * Represents a content type mapper. @@ -56,6 +57,9 @@ export interface ContentTypesMapper { */ fieldMapping: []; + entryMapping: []; + + /** * The type of the content type. */ @@ -78,11 +82,20 @@ interface ContentTypeMapperDocument { const defaultData: ContentTypeMapperDocument = { ContentTypesMappers: [] }; /** - * Represents the database instance for the content types mapper. + * Creates and returns a database instance for the content types mapper for a specific project. + * @param projectId - The unique identifier of the project + * @returns The database instance for the content types mapper */ -const db = new LowWithLodash( - new JSONFile(path.join(process.cwd(), "database", 'contentTypesMapper.json')), - defaultData -); - -export default db; +export const getContentTypesMapperDb = (projectId: string, iteration: number) => { + fs.mkdirSync(path.join(process.cwd(), "database", projectId, iteration.toString()), { recursive: true }); + const db = new LowWithLodash( + new JSONFile( + path.join(process.cwd(), "database", projectId, iteration.toString(), 'contentTypesMapper.json'), + ), + defaultData + ); + return db; +}; + +// For backward compatibility, export a default function that requires projectId +export default getContentTypesMapperDb; diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index 42d654c4d..2f4b594d0 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -70,6 +70,7 @@ interface Project { migration_execution: boolean; taxonomies?: any[]; isSSO: boolean; + iteration: number; } interface ProjectDocument { diff --git a/api/src/models/uidMapper.ts b/api/src/models/uidMapper.ts new file mode 100644 index 000000000..c4d083063 --- /dev/null +++ b/api/src/models/uidMapper.ts @@ -0,0 +1,51 @@ +import { JSONFile } from "lowdb/node"; +import LowWithLodash from "../utils/lowdb-lodash.utils.js"; +import path from "path"; +import fs from 'node:fs'; + +/** + * Represents the advanced configuration options for a field mapper. + */ +export interface Advanced { + validationRegex: string; + mandatory: boolean; + multiple: boolean; + unique: boolean; + nonLocalizable: boolean; + embedObject: boolean; + embedObjects: any; + minChars: string; + maxChars: number; + default_value: string; + description: string; + validationErrorMessage: string; + options: any[]; +} + +/** + * Represents an entry mapper object. + */ +interface EntryMapper { + entry: Record; + assets: Record; +} + +const defaultData: EntryMapper = { entry: {}, assets: {} }; + +/** + * Creates and returns a database instance for the field mapper for a specific project. + * @param projectId - The unique identifier of the project + * @returns The database instance for the field mapper + */ +const getUidMapperDb = (projectId: string, iteration: number) => { + fs.mkdirSync(path.join(process.cwd(), "database", projectId, iteration.toString()), { recursive: true }); + const db = new LowWithLodash( + new JSONFile( + path.join(process.cwd(), "database", projectId, iteration.toString(), 'uid-mapper.json') + ), + defaultData + ); + return db; +}; + +export default getUidMapperDb; diff --git a/api/src/routes/contentMapper.routes.ts b/api/src/routes/contentMapper.routes.ts index 86be4fe85..d1dcbfb4f 100644 --- a/api/src/routes/contentMapper.routes.ts +++ b/api/src/routes/contentMapper.routes.ts @@ -96,10 +96,28 @@ router.get( /** * Update content mapper - * @route GET /:orgId/:projectId + * @route PATCH /:orgId/:projectId/mapper_keys */ router.patch("/:orgId/:projectId/mapper_keys", asyncRouter(contentMapperController.updateContentMapper)); +/** + * Get Entry Mapping List + * @route GET /entryMapping/:projectId/:contentTypeId/:skip/:limit/:searchText? + */ +router.get( + "/entryMapping/:projectId/:contentTypeId/:skip/:limit/:searchText?", + asyncRouter(contentMapperController.getEntryMapping) +); + +/** + * Update Entry Status + * @route PUT /updateEntryStatus/:projectId + */ +router.put( + "/updateEntryStatus/:projectId", + asyncRouter(contentMapperController.updateEntryStatus) +); + /** * Get Single Global Field data * @route GET /:projectId/:globalFieldUid diff --git a/api/src/routes/migration.routes.ts b/api/src/routes/migration.routes.ts index 1d6d22345..d7f620b1f 100644 --- a/api/src/routes/migration.routes.ts +++ b/api/src/routes/migration.routes.ts @@ -93,5 +93,18 @@ router.post( asyncRouter(migrationController.saveMappedLocales) ) +/** + * Route for restarting the migration. + * @route POST /restart/:orgId/:projectId + * @group Migration + * @param {string} orgId - The ID of the organization. + * @param {string} projectId - The ID of the project. + * @returns {Promise} - A promise that resolves when the migration is restarted. + */ +router.post( + "/restart/:orgId/:projectId", + asyncRouter(migrationController.restartMigration) +) + export default router; diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index d2cf9ad0b..bc46345d4 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -26,8 +26,24 @@ import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; import ProjectModelLowdb from '../models/project-lowdb.js'; import FieldMapperModel from '../models/FieldMapper.js'; import { v4 as uuidv4 } from 'uuid'; -import ContentTypesMapperModelLowdb from '../models/contentTypesMapper-lowdb.js'; -import { ContentTypesMapper } from '../models/contentTypesMapper-lowdb.js'; +import getFieldMapperDb from "../models/FieldMapper.js"; +import getEntryMapperDb, { EntryMapper } from "../models/EntryMapper.js"; +import getContentTypesMapperDb, { ContentTypesMapper } from "../models/contentTypesMapper-lowdb.js"; +import getUidMapperDb from "../models/uidMapper.js"; +import { isDuplicateEntry } from '../utils/entry-duplicate.utils.js'; + + +const idCorrector = ({ id }: { id: string }) => { + const newId = id?.replace(/[-{}]/g, (match) => + match === '-' ? '' : '' + ); + if (newId) { + return newId?.toLowerCase(); + } else { + return id; + } +}; + // Developer service to create dummy contentmapping data /** @@ -37,16 +53,25 @@ import { ContentTypesMapper } from '../models/contentTypesMapper-lowdb.js'; * @returns The updated project data. */ const putTestData = async (req: Request) => { - const projectId = req.params.projectId; - const contentTypes = req.body.contentTypes; + const projectId = req?.params?.projectId; + const contentTypes = req?.body?.contentTypes; try { + // Get project data to extract iteration + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + /* this code snippet is iterating over an array called contentTypes and transforming each element by adding a unique identifier (id) if it doesn't already exist. The transformed elements are then stored in the contentType variable, and the generated id values are pushed into the contentIds array. */ + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); if (!Array?.isArray?.(contentTypes)) { throw new BadRequestError(HTTP_TEXTS.CONTENT_TYPE_INVALID); @@ -67,13 +92,12 @@ const putTestData = async (req: Request) => { if (item?.refrenceTo) { item.initialRefrenceTo = item?.refrenceTo; } - }); + }) }); const sanitizeObject = (obj: Record) => { const blockedKeys = ['__proto__', 'prototype', 'constructor']; const safeObj: Record = {}; - for (const key in obj) { if (!blockedKeys.includes(key)) { safeObj[key] = obj[key]; @@ -91,12 +115,21 @@ const putTestData = async (req: Request) => { It then updates the field_mapper property of a data object using the FieldMapperModel.update() function. Finally, it updates the fieldMapping property of each type in the contentTypes array with the fieldIds array. */ + + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await FieldMapperModel.read(); - contentTypes.forEach((type: any, index: number) => { + + // Collect all fields from all content types first + const allFields: any[] = []; + + for (let index = 0; index < contentType?.length; index++) { + const type: any = contentTypes[index]; const fieldIds: string[] = []; - const fields = Array.isArray(type?.fieldMapping) - ? type.fieldMapping.filter(Boolean).map((field: any) => { + const fields = Array.isArray(type?.fieldMapping) ? + type.fieldMapping + .filter(Boolean) + .map((field: any) => { const safeField = sanitizeObject(field); const id = safeField?.id @@ -116,12 +149,9 @@ const putTestData = async (req: Request) => { }) : []; - FieldMapperModel.update((data: any) => { - data.field_mapper = [ - ...(Array.isArray(data?.field_mapper) ? data.field_mapper : []), - ...fields, - ]; - }); + // Add to collection instead of updating DB + allFields.push(...fields); + if ( Array?.isArray?.(contentType) && Number?.isInteger?.(index) && @@ -130,13 +160,107 @@ const putTestData = async (req: Request) => { ) { contentType[index].fieldMapping = fieldIds; } + } + + // Single update with all fields + await FieldMapperModel.update((data: any) => { + data.field_mapper = allFields; + }); + + const EntryMapperModel = getEntryMapperDb(projectId, iteration); + await EntryMapperModel.read(); + + const uidMapperCurrent = getUidMapperDb(projectId, iteration); + await uidMapperCurrent.read(); + let uidMapperPrev: any = null; + if (iteration > 1) { + uidMapperPrev = getUidMapperDb(projectId, iteration - 1); + await uidMapperPrev.read(); + } + + const mergeEntry = (base: any, incoming: any) => { + const keep = { ...(base ?? {}) }; + const add = { ...(incoming ?? {}) }; + + if (!keep?.contentstackEntryUid && add?.contentstackEntryUid) { + keep.contentstackEntryUid = add.contentstackEntryUid; + } + + if (!keep?.id && add?.id) { + keep.id = add.id; + } + + Object.keys(add).forEach((k) => { + if (add[k] !== undefined) { + keep[k] = add[k]; + } + }); + + return keep; + }; + + const entryKey = (e: any) => + `${e?.contentTypeId ?? e?.contentTypeUid ?? ''}:${e?.otherCmsEntryUid ?? ''}`; + + // Collect all entries from all content types first + const allEntries: any[] = []; + + for (let index = 0; index < contentTypes?.length; index++) { + const type: any = contentTypes[index]; + const entryIds: string[] = []; + + const entries = Array.isArray(type?.entryMapping) ? + type.entryMapping + .filter(Boolean) + .map((entry: any) => { + const id = entry?.id + ? entry.id.replace(/[{}]/g, '').toLowerCase() + : uuidv4(); + entry.id = id; + entryIds.push(id); + + const otherCmsUidRaw = (entry?.otherCmsEntryUid ?? id) as string; + const uidMapperValue = resolveContentstackEntryUidAcrossIterations( + entry?.otherCmsEntryUid, + id, + uidMapperCurrent, + uidMapperPrev, + ); + + return { + ...entry, + id, + otherCmsEntryUid: entry?.otherCmsEntryUid, + projectId, + contentTypeUid: entry?.contentTypeUid ?? type?.otherCmsUid ?? type?.contentTypeUid, + contentTypeId: type?.id, + isDeleted: false, + contentstackEntryUid: uidMapperValue, + }; + }) + : []; + + // Add to collection instead of updating DB + allEntries.push(...entries); + + if ( + Array?.isArray?.(contentType) && + Number?.isInteger?.(index) && + index >= 0 && + index < contentType?.length + ) { + contentType[index].entryMapping = entryIds; + } + } + + // Single update with all entries + await EntryMapperModel.update((data: any) => { + data.entry_mapper = allEntries; }); await ContentTypesMapperModelLowdb.update((data: any) => { - data.ContentTypesMappers = [ - ...(data?.ContentTypesMappers ?? []), - ...contentType, - ]; + // Simple approach: just replace with new content types + data.ContentTypesMappers = contentType; }); await ProjectModelLowdb.read(); @@ -187,6 +311,8 @@ const putTestData = async (req: Request) => { .find({ id: projectId }) .value(); + await isDuplicateEntry(projectId); + return { status: HTTP_CODES?.OK, data: pData, @@ -229,7 +355,10 @@ const getContentTypes = async (req: Request) => { ); throw new BadRequestError(HTTP_TEXTS.PROJECT_NOT_FOUND); } - const contentMapperId = projectDetails.content_mapper; + const contentMapperId = projectDetails?.content_mapper; + const iteration = projectDetails?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); await FieldMapperModel.read(); @@ -319,6 +448,13 @@ const getFieldMapping = async (req: Request) => { let totalCount = 0; try { + const project = ProjectModelLowdb.chain + .get('projects') + .find({ id: projectId }) + .value(); + const iteration = project?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); const contentType = ContentTypesMapperModelLowdb.chain @@ -649,6 +785,9 @@ const updateContentType = async (req: Request) => { } try { + const iteration = project?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); const updateIndex = ContentTypesMapperModelLowdb.chain .get('ContentTypesMappers') @@ -826,6 +965,9 @@ const resetToInitialMapping = async (req: Request) => { throw new BadRequestError(HTTP_TEXTS.CANNOT_RESET_CONTENT_MAPPING); } + const iteration = project?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); const contentTypeData = ContentTypesMapperModelLowdb.chain .get('ContentTypesMappers') @@ -946,6 +1088,9 @@ const resetAllContentTypesMapping = async (projectId: string) => { ); throw new BadRequestError(HTTP_TEXTS.PROJECT_NOT_FOUND); } + const iteration = projectDetails?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); const cData = contentMapperId.map((cId: any) => { const contentTypeData = ContentTypesMapperModelLowdb.chain @@ -1037,7 +1182,11 @@ const removeMapping = async (projectId: string) => { ); throw new BadRequestError(HTTP_TEXTS.PROJECT_NOT_FOUND); } + const iteration = projectDetails?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); + await FieldMapperModel.read(); const cData = projectDetails?.content_mapper.map((cId: any) => { const contentTypeData = ContentTypesMapperModelLowdb.chain .get('ContentTypesMappers') @@ -1223,7 +1372,11 @@ const removeContentMapper = async (req: Request) => { ); throw new BadRequestError(HTTP_TEXTS.PROJECT_NOT_FOUND); } + const iteration = projectDetails?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); + await FieldMapperModel.read(); const cData: ContentTypesMapper[] = projectDetails?.content_mapper.map( (cId: string) => { const contentTypeData: ContentTypesMapper = @@ -1698,6 +1851,291 @@ const getExistingExtensions = async ({existingStackId, token_payload}: any) => { } +const updateEntryStatus = async (req: Request) => { + const { projectId } = req?.params; + const { ids } = req?.body; + const validatedUids: string[] = Array.isArray(ids) ? ids : []; + const srcFunc = "updateEntryMapping"; + if (isEmpty(validatedUids)) { + logger.error( + getLogMessage( + srcFunc, + "Invalid ids" + ) + ); + return { + status: HTTP_CODES?.BAD_REQUEST, + data: { + message: "Invalid ids", + }, + }; + } + try { + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + const EntryMapperModel = getEntryMapperDb(projectId, iteration); + await EntryMapperModel.read(); + const foundEntry: EntryMapper[] = []; + await EntryMapperModel.update((data: any) => { + data?.entry_mapper?.forEach((entry: any) => { + if (validatedUids.includes(entry?.id)) { + entry.isUpdate = !entry.isUpdate; + foundEntry.push(entry); + } + }); + }); + + if (foundEntry) { + return { + status: HTTP_CODES?.OK, + data: foundEntry + }; + } + + return { + status: HTTP_CODES?.NOT_FOUND, + data: { + message: "Entry not found", + }, + }; + + } catch (error: any) { + logger.error( + getLogMessage( + srcFunc, + "Error occurred while updating entry mapping", + error + ) + ); + throw new ExceptionFunction( + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR, + ); + } + + +} + +const getEntryMapping = async (req: Request) => { + + const srcFunc = "getEntryMapping"; + const contentTypeId = req?.params?.contentTypeId; + const projectId = req?.params?.projectId; + const skip: any = req?.params?.skip; + const limit: any = req?.params?.limit; + const search: string = req?.params?.searchText?.toLowerCase(); + + let result: any[] = []; + let filteredResult = []; + let totalCount = 0; + + try { + // Get project iteration + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + await ContentTypesMapperModelLowdb.read(); + + const contentType = ContentTypesMapperModelLowdb.chain + .get("ContentTypesMappers") + .find({ id: contentTypeId, projectId: projectId }) + .value(); + + if (isEmpty(contentType)) { + logger.error( + getLogMessage( + srcFunc, + `${HTTP_TEXTS.CONTENT_TYPE_NOT_FOUND} Id: ${contentTypeId}` + ) + ); + throw new BadRequestError(HTTP_TEXTS.CONTENT_TYPE_NOT_FOUND); + } + const EntryMapperModel = getEntryMapperDb(projectId, iteration); + await EntryMapperModel.read(); + let entryMapping = contentType?.entryMapping?.map?.((mapperUId: any) => { + const entryMapper = EntryMapperModel.chain + .get("entry_mapper") + .find({ id: mapperUId, projectId: projectId, contentTypeId: contentTypeId }) + .value(); + + return entryMapper; + }); + + // Fallback: If no entry mappings found in current iteration and we have previous iteration + if ((!entryMapping || entryMapping?.length === 0 || entryMapping?.every((e: any) => !e)) && iteration > 1) { + const PrevEntryMapperModel = getEntryMapperDb(projectId, iteration - 1); + await PrevEntryMapperModel.read(); + entryMapping = contentType?.entryMapping?.map?.((mapperUId: any) => { + const entryMapper = PrevEntryMapperModel.chain + .get("entry_mapper") + .find({ id: mapperUId, projectId: projectId, contentTypeId: contentTypeId }) + .value(); + + return entryMapper; + }); + } + + const enrichedMapping = await enrichEntriesWithUidMapper( + projectId, + iteration, + entryMapping ?? [], + ); + + if (!isEmpty(enrichedMapping)) { + if (search) { + filteredResult = enrichedMapping?.filter?.((item: any) => + item?.entryName?.toLowerCase().includes(search) + ); + totalCount = filteredResult?.length; + result = filteredResult?.slice(skip, Number(skip) + Number(limit)); + } else { + totalCount = enrichedMapping?.length; + result = enrichedMapping?.slice(skip, Number(skip) + Number(limit)); + } + } + return { + status: HTTP_CODES?.OK, + count: totalCount, + entryMapping: result + }; + + } catch (error: any) { + // Log error message + logger.error( + getLogMessage( + srcFunc, + "Error occurred while getting field mapping of projects", + error + ) + ); + + throw new ExceptionFunction( + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR + ); + + } +}; + +const resolveContentstackEntryUidAcrossIterations = ( + otherCmsEntryUid: string | undefined, + fallbackId: string | undefined, + currentModel: any, + prevModel: any | null, +): string | undefined => { + const fromCurrent = lookupContentstackEntryUidFromUidMap( + otherCmsEntryUid, + fallbackId, + currentModel, + ); + if (fromCurrent) return fromCurrent; + if (prevModel) { + return lookupContentstackEntryUidFromUidMap( + otherCmsEntryUid, + fallbackId, + prevModel, + ); + } + return undefined; +}; + +const lookupContentstackEntryUidFromUidMap = ( + otherCmsEntryUid: string | undefined, + fallbackId: string | undefined, + uidMapperModel: any, +): string | undefined => { + const map = getEntryUidMap(uidMapperModel); + const otherCmsUidRaw = (otherCmsEntryUid ?? fallbackId ?? '') as string; + if (!otherCmsUidRaw) return undefined; + const otherCmsUid = otherCmsUidRaw.replace(/[{}]/g, ''); + const otherCmsUidLower = otherCmsUid ? otherCmsUid.toLowerCase() : ''; + const resolved = + map[otherCmsUid] || + map[otherCmsUidRaw] || + map[otherCmsUidLower] || + map[idCorrector({ id: otherCmsUid })] || + (otherCmsUidLower ? map[idCorrector({ id: otherCmsUidLower })] : undefined); + if (resolved == null || resolved === '' || resolved === ' ') return undefined; + return String(resolved).trim() || undefined; +}; + +const getEntryUidMap = (uidMapperModel: any): Record => { + const d = uidMapperModel?.data ?? {}; + const pick = (x: unknown): Record => { + if (!x || typeof x !== 'object' || Array.isArray(x)) return {}; + return x as Record; + }; + const fromEntryUid = flattenNestedUidMap(pick(d.entryUid)); + const fromEntry = flattenNestedUidMap(pick(d.entry)); + const nUid = Object?.keys(fromEntryUid).length; + const nEnt = Object?.keys(fromEntry).length; + if (nUid > 0 && nEnt > 0) { + return { ...fromEntry, ...fromEntryUid }; + } + if (nUid > 0) return fromEntryUid; + if (nEnt > 0) return fromEntry; + return {}; +}; + +const flattenNestedUidMap = (raw: Record): Record => { + const keys = Object?.keys(raw ?? {}); + if (keys?.length === 0) return {}; + const nested = keys?.every((k) => { + const v = raw[k]; + return v != null && typeof v === 'object' && !Array.isArray(v); + }); + if (!nested) return { ...raw }; + return keys.reduce>((acc, k) => ({ ...acc, ...raw[k] }), {}); +}; + +/** + * Fill missing contentstackEntryUid from uid-mapper. Uses **current** iteration first + * (where the latest CLI import writes), then iteration-1 so step 3 still works right + * after restart before a re-import. + */ +const enrichEntriesWithUidMapper = async ( + projectId: string, + iteration: number, + entries: any[], +): Promise => { + if (!Array.isArray(entries) || entries?.length === 0) return entries; + + const currentModel = getUidMapperDb(projectId, iteration); + await currentModel.read(); + + let prevModel: any = null; + if (iteration > 1) { + prevModel = getUidMapperDb(projectId, iteration - 1); + await prevModel.read(); + } + + return entries?.map((item: any) => { + if (!item) return item; + const existing = item?.contentstackEntryUid; + if (existing != null && String(existing).trim() !== '' && existing !== ' ') { + return item; + } + const resolved = resolveContentstackEntryUidAcrossIterations( + item?.otherCmsEntryUid, + item?.id, + currentModel, + prevModel, + ); + return resolved ? { ...item, contentstackEntryUid: resolved } : item; + }); +}; + + + export const contentMapperService = { putTestData, getContentTypes, @@ -1714,4 +2152,6 @@ export const contentMapperService = { getSingleGlobalField, getExistingTaxonomies, getExistingExtensions, + getEntryMapping, + updateEntryStatus, }; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 16b9ff5e0..b1a85116a 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -17,10 +17,12 @@ import { STEPPER_STEPS, CMS, GET_AUDIT_DATA, + MIGRATION_DATA_CONFIG, } from '../constants/index.js'; import { BadRequestError, ExceptionFunction, + NotFoundError, } from '../utils/custom-errors.utils.js'; import { fieldAttacher } from '../utils/field-attacher.utils.js'; import { siteCoreService } from './sitecore.service.js'; @@ -38,12 +40,20 @@ import fsPromises from 'fs/promises'; import { matchesSearchText } from '../utils/search.util.js'; import { taxonomyService } from './taxonomy.service.js'; import { globalFieldServie } from './globalField.service.js'; -import { getSafePath, sanitizeStackId } from '../utils/sanitize-path.utils.js'; +import { + assertResolvedPathUnderBase, + getSafePath, + sanitizeProjectId, + sanitizeStackId, +} from '../utils/sanitize-path.utils.js'; import { aemService } from './aem.service.js'; import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; +import { utilsUpdateCli } from './updateEntryCli.service.js'; +import { enrichConfigWithAssetMapping, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; +import { removeExistingAssets, saveAssetMetadata } from '../utils/asset-update.utils.js'; /** - * Creates a test stack. + * Creates a test stack. * * @param req - The request object containing the necessary parameters. * @returns A promise that resolves to a LoginServiceType object. @@ -297,12 +307,20 @@ const startTestMigration = async (req: Request): Promise => { const { legacy_cms: { cms, file_path }, } = project; + const logsBase = path.resolve(process.cwd(), 'logs'); + const safeTestProjectId = sanitizeProjectId(projectId); + const safeTestStackId = sanitizeStackId(project?.current_test_stack_id); + if (!safeTestProjectId || !safeTestStackId) { + throw new BadRequestError( + 'Invalid project or test stack identifier; cannot create log file path.' + ); + } const loggerPath = path.join( - process.cwd(), - 'logs', - projectId, - `${project?.current_test_stack_id}.log` + logsBase, + safeTestProjectId, + `${safeTestStackId}.log` ); + assertResolvedPathUnderBase(logsBase, loggerPath); const message = getLogMessage( 'startTestMigration', 'Starting Test Migration...', @@ -410,8 +428,8 @@ const startTestMigration = async (req: Request): Promise => { await copyLogsToTestStack(project?.current_test_stack_id, loggerPath); const contentTypes = await fieldAttacher({ orgId, - projectId, - destinationStackId: project?.current_test_stack_id, + projectId: safeTestProjectId, + destinationStackId: safeTestStackId, region, user_id, is_sso, @@ -702,12 +720,20 @@ const startMigration = async (req: Request): Promise => { const { legacy_cms: { cms, file_path }, } = project; + const logsBase = path.resolve(process.cwd(), 'logs'); + const safeFinalProjectId = sanitizeProjectId(projectId); + const safeFinalStackId = sanitizeStackId(project?.destination_stack_id); + if (!safeFinalProjectId || !safeFinalStackId) { + throw new BadRequestError( + 'Invalid project or destination stack identifier; cannot create log file path.' + ); + } const loggerPath = path.join( - process.cwd(), - 'logs', - projectId, - `${project?.destination_stack_id}.log` + logsBase, + safeFinalProjectId, + `${safeFinalStackId}.log` ); + assertResolvedPathUnderBase(logsBase, loggerPath); const message = getLogMessage( 'start Migration', 'Starting Migration...', @@ -817,8 +843,8 @@ const startMigration = async (req: Request): Promise => { const contentTypes = await fieldAttacher({ orgId, - projectId, - destinationStackId: project?.destination_stack_id, + projectId: safeFinalProjectId, + destinationStackId: safeFinalStackId, region, user_id, is_sso, @@ -1062,6 +1088,102 @@ const startMigration = async (req: Request): Promise => { default: break; } + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + let configFilePath: string | null = null; + let safeDeltaMigrationLogPath: string | undefined; + + const safeStackForAssets = sanitizeStackId(project?.destination_stack_id); + if (!safeStackForAssets) { + console.error( + 'Invalid destination stack id; cannot load assets index.', + ); + return; + } + const migrationDataBase = path.resolve( + process.cwd(), + MIGRATION_DATA_CONFIG.DATA, + ); + const assetsDir = path.join( + migrationDataBase, + safeStackForAssets, + MIGRATION_DATA_CONFIG.ASSETS_DIR_NAME, + ); + const indexPath = path.join( + assetsDir, + MIGRATION_DATA_CONFIG.ASSETS_SCHEMA_FILE, + ); + + let indexData: Record; + try { + assertResolvedPathUnderBase(migrationDataBase, indexPath); + } catch { + console.error( + 'Assets index path is outside the allowed migration-data directory.', + ); + return; + } + + try { + const stats = await fsPromises.lstat(indexPath).catch(() => null); + if (!stats || stats.isSymbolicLink() || !stats.isFile()) { + console.error( + `Assets index not found or not a regular file at ${indexPath}`, + ); + return; + } + + const canonicalIndexPath = await fsPromises.realpath(indexPath); + try { + assertResolvedPathUnderBase(migrationDataBase, canonicalIndexPath); + } catch { + console.error( + 'Assets index resolves outside the allowed migration-data directory.', + ); + return; + } + + const raw = await fsPromises.readFile(canonicalIndexPath, 'utf-8'); + if (!raw?.trim()) { + console.error(`Assets index.json is empty at ${indexPath}`); + return; + } + indexData = JSON.parse(raw); + } catch (error) { + console.error( + `Failed to read or parse assets index.json at ${indexPath}:`, + error instanceof Error ? error.message : String(error), + ); + return; + } + + saveAssetMetadata(indexData, projectId, iteration, safeDeltaMigrationLogPath); + + if (iteration > 1) { + const logsBase = path.resolve(process.cwd(), 'logs'); + const safePid = sanitizeProjectId(projectId); + const safeStack = sanitizeStackId(project?.destination_stack_id); + if (safePid && safeStack) { + const candidate = path.join(logsBase, safePid, `${safeStack}.log`); + try { + assertResolvedPathUnderBase(logsBase, candidate); + safeDeltaMigrationLogPath = candidate; + } catch { + safeDeltaMigrationLogPath = undefined; + } + } + await removeExistingAssets(projectId, safeDeltaMigrationLogPath); + configFilePath = await removeEntriesFromDatabase( + projectId, + safeDeltaMigrationLogPath + ); + console.info('Config file written to:', configFilePath); + } + await utilsCli?.runCli( region, user_id, @@ -1070,6 +1192,23 @@ const startMigration = async (req: Request): Promise => { false, loggerPath ); + + if (configFilePath) { + console.info('Config file path:', configFilePath); + enrichConfigWithAssetMapping( + configFilePath, + projectId, + iteration, + safeDeltaMigrationLogPath + ); + await utilsUpdateCli?.updateEntryCli( + region, + user_id, + project?.destination_stack_id, + safeDeltaMigrationLogPath || '', + configFilePath + ); + } } }; const getAuditData = async (req: Request): Promise => { @@ -1514,6 +1653,37 @@ export const updateLocaleMapper = async (req: Request) => { } }; +const restartMigration = async (req: Request): Promise => { + const { orgId, projectId } = req?.params ?? {}; + await ProjectModelLowdb.read(); + const projectIndex = ProjectModelLowdb.chain + .get("projects") + .findIndex({ id: projectId, org_id: orgId }) + .value(); + console.info('projectIndex', projectIndex); + if (projectIndex > -1) { + await ProjectModelLowdb.update((data: any) => { + data.projects[projectIndex].migration_execution = false; + data.projects[projectIndex].isMigrationCompleted = false; + data.projects[projectIndex].isMigrationStarted = false; + data.projects[projectIndex].current_step = 1; + data.projects[projectIndex].status = 0; + data.projects[projectIndex].legacy_cms = { + ...data.projects[projectIndex].legacy_cms, + is_fileValid: false, + }; + data.projects[projectIndex].iteration = 1 + (data.projects[projectIndex].iteration || 0); + data.projects[projectIndex].updated_at = new Date().toISOString(); + }); + } else { + throw new NotFoundError(HTTP_TEXTS?.PROJECT_NOT_FOUND); + } + return { + status: HTTP_CODES?.OK, + message: "Migration restarted successfully", + }; +}; + export const migrationService = { createTestStack, deleteTestStack, @@ -1523,4 +1693,5 @@ export const migrationService = { createSourceLocales, updateLocaleMapper, getAuditData, -}; + restartMigration +}; \ No newline at end of file diff --git a/api/src/services/projects.service.ts b/api/src/services/projects.service.ts index 0d426570e..8e5524022 100644 --- a/api/src/services/projects.service.ts +++ b/api/src/services/projects.service.ts @@ -2,7 +2,7 @@ import { Request } from 'express'; import ProjectModelLowdb from '../models/project-lowdb.js'; -import ContentTypesMapperModelLowdb from '../models/contentTypesMapper-lowdb.js'; +import ContentTypesMapperModelLowdb, { getContentTypesMapperDb } from '../models/contentTypesMapper-lowdb.js'; import FieldMapperModel from '../models/FieldMapper.js'; import { @@ -26,6 +26,7 @@ import logger from "../utils/logger.js"; import AuthenticationModel from "../models/authentication.js"; // import { contentMapperService } from "./contentMapper.service.js"; import { v4 as uuidv4 } from 'uuid'; +import getFieldMapperDb from '../models/FieldMapper.js'; /** * Retrieves all projects based on the provided request object. @@ -163,6 +164,7 @@ const createProject = async (req: Request) => { isMigrationCompleted:false, migration_execution:false, isSSO: isSSO, + iteration: 1, }; try { @@ -1155,6 +1157,9 @@ const deleteProject = async (req: Request) => { if (projects?.status == NEW_PROJECT_STATUS[5]) { const content_mapper_id = projects?.content_mapper; + const iteration = projects?.iteration || 0; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration); + const FieldMapperModel = getFieldMapperDb(projectId, iteration); await ContentTypesMapperModelLowdb.read(); await FieldMapperModel.read(); if (!isEmpty(content_mapper_id) && Array.isArray(content_mapper_id)) { diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index 5a27db4e2..99ba39908 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -19,6 +19,7 @@ interface TestStack { isMigrated: boolean; } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; +import getUidMapperDb from '../models/uidMapper.js'; /** * Determines log level based on message content without removing ANSI codes @@ -53,6 +54,67 @@ const stripAnsiCodes = (text: string): string => { return text.replace(/\u001b\[\d+m/g, ''); }; +const writeUidMapping = async (backupPath: string, projectId: string, iteration: number) => { + try { + const assetMapperPath = path.join(backupPath, 'mapper', 'assets', 'uid-mapping.json'); + let assetJson = {}; + + // Check if file exists and has meaningful data + if (fs.existsSync(assetMapperPath)) { + const assetData = fs.readFileSync(assetMapperPath, 'utf-8'); + const parsedData = JSON.parse(assetData); + // Check if data is not empty + if (parsedData && Object?.keys(parsedData)?.length > 0) { + assetJson = parsedData; + } + } + + // If no meaningful data found and we have previous iteration, use fallback + if (Object?.keys(assetJson)?.length === 0 && iteration > 1) { + const prevAssetMapperPath = path.join(process.cwd(), 'database', projectId, (iteration - 1).toString(), 'uid-mapper.json'); + if (fs.existsSync(prevAssetMapperPath)) { + const prevData = JSON.parse(fs.readFileSync(prevAssetMapperPath, 'utf-8')); + assetJson = prevData?.assets || {}; + } + } + + const entryMapperPath = path.join(backupPath, 'mapper', 'entries', 'uid-mapping.json'); + let entryJson = {}; + + // Check if file exists and has meaningful data + if (fs.existsSync(entryMapperPath)) { + const entryData = fs.readFileSync(entryMapperPath, 'utf-8'); + const parsedData = JSON.parse(entryData); + // Check if data is not empty + if (parsedData && Object?.keys(parsedData)?.length > 0) { + entryJson = parsedData; + } + } + + // If no meaningful data found and we have previous iteration, use fallback + if (Object?.keys(entryJson)?.length === 0 && iteration > 1) { + const prevEntryMapperPath = path.join(process.cwd(), 'database', projectId, (iteration - 1).toString(), 'uid-mapper.json'); + if (fs.existsSync(prevEntryMapperPath)) { + const prevData = JSON.parse(fs.readFileSync(prevEntryMapperPath, 'utf-8')); + console.info('Using previous iteration data for entries:', prevData); + entryJson = prevData?.entry || {}; + } + } + + const combinedMapping = { + assets: assetJson, + entry: entryJson, + }; + const UidMapperModelLowdb = getUidMapperDb(projectId, iteration); + await UidMapperModelLowdb.read(); + UidMapperModelLowdb.data = combinedMapping; + await UidMapperModelLowdb.write(); + console.info('UID mapping data written successfully to Lowdb'); + } catch (error) { + console.error('Error writing UID mapping file:', error); + } +} + /** * Executes CLI commands and provides real-time output * Uses Node's spawn to run commands asynchronously @@ -269,6 +331,13 @@ export const runCli = async ( if (loggerPath && loggerPath !== transformePath) { fs.appendFileSync(loggerPath, JSON.stringify(directLogEntry) + '\n'); } + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + await writeUidMapping(backupPath, projectId, iteration); } // Keep the project status update code: @@ -321,4 +390,4 @@ export const runCli = async ( } }; -export const utilsCli = { runCli }; +export const utilsCli = { runCli }; \ No newline at end of file diff --git a/api/src/services/updateEntryCli.service.ts b/api/src/services/updateEntryCli.service.ts new file mode 100644 index 000000000..fb1faf346 --- /dev/null +++ b/api/src/services/updateEntryCli.service.ts @@ -0,0 +1,252 @@ +/* eslint-disable */ + +import path from 'path'; +import fs from 'fs'; +import { spawn } from 'child_process'; +import { CS_REGIONS } from '../constants/index.js'; +import AuthenticationModel from '../models/authentication.js'; +import { setLogFilePath } from '../server.js'; +// import utilitiesHandler from '@contentstack/cli-utilities'; +import { setOAuthConfig } from '../utils/config-handler.util.js'; +import { setBasicAuthConfig } from '../utils/config-handler.util.js'; + +const determineLogLevel = (text: string): string => { + const lowerText = text.toLowerCase(); + + if ( + lowerText.includes('error') || + lowerText.includes('failed') || + lowerText.includes('exception') || + lowerText.includes('not found') + ) { + return 'error'; + } else if (lowerText.includes('warn') || lowerText.includes('warning')) { + return 'warn'; + } else { + return 'info'; + } +}; + +const stripAnsiCodes = (text: string): string => { + return text.replace(/\u001b\[\d+m/g, ''); +}; + +const runCommand = ( + command: string, + args: string[] = [], + logFilePath?: string, +): Promise => { + return new Promise((resolve, reject) => { + const cmdProcess = spawn(command, args, { shell: true }); + + cmdProcess.stdout.on('data', (data) => { + const output = data.toString(); + process.stdout.write(output); + + if (logFilePath) { + try { + const cleanedOutput = stripAnsiCodes(output); + const logEntry = { + level: determineLogLevel(cleanedOutput), + message: cleanedOutput.trim(), + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(logEntry) + '\n'); + } catch (err) { + console.error('Error writing to log file:', err); + } + } + }); + + cmdProcess.stderr.on('data', (data) => { + const output = data.toString(); + process.stderr.write(output); + + if (logFilePath) { + try { + const cleanedOutput = stripAnsiCodes(output); + const logEntry = { + level: 'error', + message: cleanedOutput.trim(), + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(logEntry) + '\n'); + } catch (err) { + console.error('Error writing stderr to log file:', err); + } + } + }); + + cmdProcess.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + if (logFilePath) { + try { + const logEntry = { + level: 'error', + message: `Command failed with exit code ${code}`, + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(logEntry) + '\n'); + } catch (err) { + console.error('Error writing close event to log file:', err); + } + } + reject(new Error(`Command failed with exit code ${code}`)); + } + }); + }); +}; + +/** + * Runs csdx cm:stacks:migration to execute the entry update script + * against the target stack. + */ +export const updateEntryCli = async ( + rg: string, + user_id: string, + stack_api_key: string, + logFilePath: string, + configFilePath: string +) => { + try { + console.info("inside updateEntryCli"); + const regionPresent = + CS_REGIONS.find((item) => item === rg) ?? 'NA'.replace(/_/g, '-'); + const regionCli = regionPresent.replace(/_/g, '-'); + + const directLogEntry1 = { + level: 'info', + message: `Starting entry update CLI process for stack: ${stack_api_key}`, + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry1) + '\n'); + + await AuthenticationModel.read(); + const userData = AuthenticationModel.chain + .get('users') + .find({ region: regionPresent, user_id }) + .value(); + + await runCommand( + 'npx', + ['@contentstack/cli', 'config:set:region', regionCli], + logFilePath + ); + + // utilitiesHandler.configHandler.set('authtoken', userData.authtoken); + // utilitiesHandler.configHandler.set('email', userData.email); + // utilitiesHandler.configHandler.set('authorisationType', 'BASIC'); + if(userData?.access_token){ + setOAuthConfig(userData); + + }else if(userData?.authtoken){ + setBasicAuthConfig(userData); + }else { + throw new Error("No authentication token found"); + } + const hasAuth = Boolean(userData?.authtoken || userData?.access_token); + if (!hasAuth || !stack_api_key) { + const directLogEntry2 = { + level: 'info', + message: 'User not found, no auth token (authtoken or access_token), or stack API key missing.', + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry2) + '\n'); + return; + } + + const directLogEntry3 = { + level: 'info', + message: `Authentication configured for user: ${userData?.email}`, + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry3) + '\n'); + + const scriptPath = path.join( + process.cwd(), + 'src', + 'utils', + 'entry-update-script.cjs' + ); + const directLogEntryScript = { + level: 'info', + message: `Script path: ${scriptPath}`, + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntryScript) + '\n'); + + await setLogFilePath(logFilePath); + + const directLogEntry4 = { + level: 'info', + message: 'Running update entry migration script', + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry4) + '\n'); + + const directLogEntry5 = { + level: 'info', + message: `Updating entries using config file: ${configFilePath}`, + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry5) + '\n'); + + await runCommand( + 'npx', + [ + '@contentstack/cli', + 'cm:stacks:migration', + '--file-path', + scriptPath.includes(' ') ? `"${scriptPath}"` : scriptPath, + '-k', + stack_api_key, + '--config-file', + configFilePath.includes(' ') ? `"${configFilePath}"` : configFilePath, + ], + logFilePath, + ); + + const directLogEntry6 = { + level: 'info', + message: 'Entry update migration completed successfully', + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry6) + '\n'); + + const directLogEntry7 = { + level: 'info', + message: `All entries have been updated in Contentstack stack: ${stack_api_key}`, + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry7) + '\n'); + + const directLogEntry = { + level: 'info', + message: 'Entry Update Process Completed', + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry) + '\n'); + } catch (error) { + console.error('updateEntryCli error:', error); + const directLogEntry8 = { + level: 'error', + message: `Failed to update entries for stack: ${stack_api_key}`, + methodName: 'updateEntryCli', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(logFilePath, JSON.stringify(directLogEntry8) + '\n'); + } +}; + +export const utilsUpdateCli = { updateEntryCli }; diff --git a/api/src/utils/asset-update.utils.ts b/api/src/utils/asset-update.utils.ts new file mode 100644 index 000000000..637b2aa55 --- /dev/null +++ b/api/src/utils/asset-update.utils.ts @@ -0,0 +1,342 @@ +import ProjectModelLowdb from "../models/project-lowdb.js"; +import path from "path"; +import fs from "node:fs"; +import { MIGRATION_DATA_CONFIG } from "../constants/index.js"; + +/** + * Helper function to write log entries to file + */ +const writeLogEntry = (message: string, methodName: string, loggerPath?: string) => { + if (loggerPath) { + const directLogEntry = { + level: 'info', + message, + methodName, + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(loggerPath, JSON.stringify(directLogEntry) + '\n'); + } +}; + +interface AssetMetadata { + filename: string; + file_size: string; + url: string; +} + +/** + * Traverses an object and replaces any asset reference whose uid matches + * a source asset ID with the corresponding Contentstack asset UID. + * Asset references are objects with a "uid" property matching a known source asset ID. + */ +const replaceAssetRefsInObject = ( + obj: any, + assetUidMap: Map +): boolean => { + if (!obj || typeof obj !== "object") return false; + + let modified = false; + + for (const key of Object.keys(obj)) { + const value = obj[key]; + if (!value || typeof value !== "object") continue; + + if (value?.uid && assetUidMap?.has(value?.uid)) { + obj[key] = assetUidMap?.get(value?.uid); + modified = true; + } else { + const childModified = replaceAssetRefsInObject(value, assetUidMap); + if (childModified) modified = true; + } + } + + return modified; +}; + +/** + * Saves asset metadata from index.json to database/{projectId}/{iteration}/asset-metadata.json. + * Used for validation in subsequent iterations. + */ +export const saveAssetMetadata = ( + indexData: Record, + projectId: string, + iteration: number, + loggerPath?: string +): void => { + const metadata: Record = {}; + + for (const [assetId, asset] of Object.entries(indexData)) { + metadata[assetId] = { + filename: asset?.filename || "", + file_size: asset?.file_size || "", + url: asset?.url || "", + }; + } + + const metadataDir = path.join(process.cwd(), "database", projectId, iteration.toString()); + fs.mkdirSync(metadataDir, { recursive: true }); + const metadataPath = path.join(metadataDir, "asset-metadata.json"); + fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2), "utf-8"); + writeLogEntry(`Asset metadata saved: ${Object?.keys(metadata)?.length} assets → ${metadataPath}`, "saveAssetMetadata", loggerPath); +}; + +/** + * Loads asset metadata from a previous iteration. + */ +const loadPreviousAssetMetadata = ( + projectId: string, + prevIteration: number +): Record => { + const metadataPath = path.join( + process.cwd(), "database", projectId, + prevIteration.toString(), "asset-metadata.json" + ); + if (!fs.existsSync(metadataPath)) { + // Note: This function doesn't have access to loggerPath, keeping as console for internal function + console.info(`No previous asset metadata found at ${metadataPath}`); + return {}; + } + try { + return JSON.parse(fs.readFileSync(metadataPath, "utf-8")); + } catch (err) { + console.error("Failed to read previous asset metadata:", err); + return {}; + } +}; + +/** + * Loads the asset uid mapping from a previous iteration's uid-mapper.json. + */ +const loadPreviousAssetUidMap = ( + projectId: string, + prevIteration: number +): Record => { + const uidMapperPath = path.join( + process.cwd(), "database", projectId, + prevIteration.toString(), "uid-mapper.json" + ); + if (!fs.existsSync(uidMapperPath)) { + // Note: This function doesn't have access to loggerPath, keeping as console for internal function + console.info(`No uid-mapper found at ${uidMapperPath}`); + return {}; + } + try { + const data = JSON.parse(fs.readFileSync(uidMapperPath, "utf-8")); + return data?.assets || {}; + } catch (err) { + console.error("Failed to read uid-mapper:", err); + return {}; + } +}; + +/** + * Determines whether an asset has changed by comparing current metadata + * against the previous iteration's stored metadata. + */ +const hasAssetChanged = ( + assetId: string, + currentAsset: any, + prevMetadata: Record +): boolean => { + const prev = prevMetadata[assetId]; + if (!prev) return true; + + const currentFilename = currentAsset?.filename || ""; + const currentFileSize = currentAsset?.file_size || ""; + + if (currentFilename !== prev.filename || currentFileSize !== prev.file_size) { + // Note: This function doesn't have access to loggerPath, keeping as console for internal function + console.info( + `Asset "${assetId}" changed: ` + + `filename "${prev.filename}" → "${currentFilename}", ` + + `file_size "${prev.file_size}" → "${currentFileSize}"` + ); + return true; + } + + return false; +}; + +/** + * Removes existing (already-migrated) assets from cmsMigrationData to prevent duplicates. + * + * For iteration 1: only saves asset metadata for future comparisons. + * For iteration 2+: + * 1. Reads uid-mapper.assets from previous iteration + * 2. Reads asset-metadata.json from previous iteration + * 3. For each asset in current index.json: + * - If NOT in uid-mapper → new asset, keep it + * - If in uid-mapper AND metadata matches → unchanged, replace refs with CS UID, remove from import + * - If in uid-mapper BUT metadata differs → updated asset, keep it for re-import + * 4. Replaces asset references in entry JSON files with Contentstack UIDs + * 5. Removes deduplicated asset entries from index.json and their file folders + * 6. Saves current asset metadata for the next iteration + */ +export const removeExistingAssets = async (projectId: string, loggerPath?: string): Promise => { + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + + const iteration = projectData?.iteration || 1; + const stackId = projectData?.destination_stack_id; + + if (!stackId) { + writeLogEntry("No stackId found, skipping asset dedup.", "removeExistingAssets", loggerPath); + return; + } + + const assetsDir = path.join( + process.cwd(), MIGRATION_DATA_CONFIG.DATA, stackId, + MIGRATION_DATA_CONFIG.ASSETS_DIR_NAME + ); + const indexPath = path.join(assetsDir, MIGRATION_DATA_CONFIG.ASSETS_SCHEMA_FILE); + if (!fs.existsSync(indexPath)) { + writeLogEntry(`Assets index.json not found at ${indexPath}, skipping.`, "removeExistingAssets", loggerPath); + return; + } + writeLogEntry(`Assets index.json found at ${indexPath}`, "removeExistingAssets", loggerPath); + + let indexData: Record; + try { + const raw = fs.readFileSync(indexPath, "utf-8"); + if (!raw.trim()) { + console.error(`Assets index.json is empty at ${indexPath}`); + return; + } + indexData = JSON.parse(raw); + } catch (error) { + console.error(`Failed to parse assets index.json at ${indexPath}:`, error instanceof Error ? error.message : String(error)); + return; + } + + saveAssetMetadata(indexData, projectId, iteration, loggerPath); + + if (iteration <= 1) { + writeLogEntry("Iteration 1: asset metadata saved, no dedup needed.", "removeExistingAssets", loggerPath); + return; + } + writeLogEntry(`Iteration ${iteration} found, loading previous asset uid map and metadata.`, "removeExistingAssets", loggerPath); + + const prevIteration = iteration - 1; + const prevAssetUidMap = loadPreviousAssetUidMap(projectId, prevIteration); + writeLogEntry(`Previous asset uid map loaded from ${prevIteration} iteration.`, "removeExistingAssets", loggerPath); + const prevMetadata = loadPreviousAssetMetadata(projectId, prevIteration); + + if (!Object?.keys(prevAssetUidMap)?.length) { + writeLogEntry("No previous asset uid mapping found, skipping dedup.", "removeExistingAssets", loggerPath); + return; + } + writeLogEntry(`Previous asset metadata loaded from ${prevIteration} iteration.`, "removeExistingAssets", loggerPath); + const assetsToReuse = new Map(); + const assetsToRemoveFromIndex: string[] = []; + + for (const [assetId, assetData] of Object.entries(indexData)) { + const contentstackUid = prevAssetUidMap[assetId]; + if (!contentstackUid) continue; + + if (!hasAssetChanged(assetId, assetData, prevMetadata)) { + assetsToReuse.set(assetId, contentstackUid); + assetsToRemoveFromIndex.push(assetId); + writeLogEntry(`Asset "${assetId}" unchanged → reuse CS UID "${contentstackUid}"`, "removeExistingAssets", loggerPath); + writeLogEntry(`Asset "${assetId}" has been reused from previous migration`, "removeExistingAssets", loggerPath); + } else { + writeLogEntry(`Asset "${assetId}" changed → will re-import`, "removeExistingAssets", loggerPath); + } + } + + if (!assetsToReuse?.size) { + writeLogEntry("No unchanged assets to deduplicate.", "removeExistingAssets", loggerPath); + return; + } + + // 1. Replace asset references in entry JSON files + const entriesDir = path.join( + process.cwd(), MIGRATION_DATA_CONFIG.DATA, stackId, + MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME + ); + + if (fs.existsSync(entriesDir)) { + const contentTypeDirs = fs.readdirSync(entriesDir, { withFileTypes: true }) + ?.filter((d) => d?.isDirectory()); + + for (const ctDir of contentTypeDirs) { + const ctPath = path.join(entriesDir, ctDir?.name); + + if (!fs.existsSync(ctPath)) { + console.warn(`Content type directory not found: ${ctPath}`); + continue; + } + + const localeDirs = fs.readdirSync(ctPath, { withFileTypes: true }) + ?.filter((d) => d?.isDirectory()); + + for (const localeDir of localeDirs) { + const localePath = path.join(ctPath, localeDir?.name); + + if (!fs.existsSync(localePath)) { + console.warn(`Locale directory not found: ${localePath}`); + continue; + } + + const jsonFiles = fs.readdirSync(localePath) + .filter((f) => f.endsWith(".json") && f !== "index.json"); + + for (const jsonFile of jsonFiles) { + const filePath = path.join(localePath, jsonFile); + + try { + const raw = fs.readFileSync(filePath, "utf-8"); + + // Check if file is empty or contains only whitespace + if (!raw.trim()) { + console.warn(`Skipping empty file: ${filePath}`); + continue; + } + + const data = JSON.parse(raw); + + const modified = replaceAssetRefsInObject(data, assetsToReuse); + if (modified) { + fs.writeFileSync(filePath, JSON.stringify(data), "utf-8"); + writeLogEntry(`Replaced asset refs in ${filePath}`, "removeExistingAssets", loggerPath); + } + } catch (error) { + console.error(`Failed to process file ${filePath}:`, error instanceof Error ? error.message : String(error)); + console.warn(`Skipping problematic file: ${filePath}`); + continue; // Skip this file and continue with others + } + } + } + } + } + + // 2. Remove deduplicated assets from index.json + for (const assetId of assetsToRemoveFromIndex) { + delete indexData[assetId]; + writeLogEntry(`Asset "${assetId}" has been removed from migration data (already exists in Contentstack)`, "removeExistingAssets", loggerPath); + } + fs.writeFileSync(indexPath, JSON.stringify(indexData, null, 4), "utf-8"); + writeLogEntry(`Removed ${assetsToRemoveFromIndex?.length} assets from index.json`, "removeExistingAssets", loggerPath); + + // 3. Remove asset file folders + const filesDir = path.join(assetsDir, "files"); + if (fs.existsSync(filesDir)) { + for (const assetId of assetsToRemoveFromIndex) { + const assetFolder = path.join(filesDir, assetId); + if (fs.existsSync(assetFolder)) { + fs.rmSync(assetFolder, { recursive: true, force: true }); + writeLogEntry(`Removed asset folder: ${assetFolder}`, "removeExistingAssets", loggerPath); + writeLogEntry(`Asset "${assetId}" physical files have been removed from migration data`, "removeExistingAssets", loggerPath); + } + } + } + + writeLogEntry( + `Asset dedup complete: ${assetsToReuse.size} reused, ` + + `${Object?.keys(indexData)?.length} remaining for import.`, + "removeExistingAssets", + loggerPath + ); +}; diff --git a/api/src/utils/content-type-checker.utils.ts b/api/src/utils/content-type-checker.utils.ts new file mode 100644 index 000000000..5797a8b29 --- /dev/null +++ b/api/src/utils/content-type-checker.utils.ts @@ -0,0 +1,122 @@ +import path from 'path'; +import fs from 'fs'; +import getContentTypesMapperDb from '../models/contentTypesMapper-lowdb.js'; + +/** + * Checks if a content type has already been created in any previous iteration. + * This prevents duplicate content type creation during delta migrations. + * + * @param projectId - The project ID to check + * @param contentTypeUid - The content type UID (otherCmsUid) to check + * @param currentIteration - The current iteration number + * @returns true if content type already exists in previous iterations, false otherwise + */ +export const isContentTypeAlreadyCreated = async ( + projectId: string, + contentTypeUid: string, + currentIteration: number +): Promise => { + // For iteration 1, no previous iterations exist + if (currentIteration <= 1) { + return false; + } + + // Check all previous iterations (1 to currentIteration-1) + for (let i = 1; i < currentIteration; i++) { + try { + // Check if iteration directory exists + const iterationPath = path.join(process.cwd(), 'database', projectId, i.toString()); + if (!fs.existsSync(iterationPath)) { + continue; // Skip missing iterations + } + + // Read the contentTypesMapper database for this iteration + const contentTypesMapperDb = getContentTypesMapperDb(projectId, i); + await contentTypesMapperDb.read(); + + // Check if any content type matches the given UID + const contentTypes = contentTypesMapperDb.data?.ContentTypesMappers || []; + const exists = contentTypes.some( + (ct: any) => ct.otherCmsUid === contentTypeUid + ); + + if (exists) { + console.info(`Content type '${contentTypeUid}' already created in iteration ${i}`); + return true; + } + } catch (error) { + console.warn(`Failed to check iteration ${i} for content type '${contentTypeUid}' : `, error); + // Continue checking other iterations even if one fails + continue; + } + } + + return false; +}; + +/** + * Gets all content type UIDs that have been created in previous iterations. + * Useful for bulk checking or debugging purposes. + * + * @param projectId - The project ID to check + * @param currentIteration - The current iteration number + * @returns Array of content type UIDs that already exist + */ +export const getPreviouslyCreatedContentTypes = async ( + projectId: string, + currentIteration: number +): Promise => { + const existingContentTypes = new Set(); + + // For iteration 1, no previous iterations exist + if (currentIteration <= 1) { + return []; + } + + // Check all previous iterations + for (let i = 1; i < currentIteration; i++) { + try { + // Check if iteration directory exists + const iterationPath = path.join(process.cwd(), 'database', projectId, i.toString()); + if (!fs.existsSync(iterationPath)) { + continue; // Skip missing iterations + } + + // Read the contentTypesMapper database for this iteration + const contentTypesMapperDb = getContentTypesMapperDb(projectId, i); + await contentTypesMapperDb.read(); + + + // Collect all content type UIDs from this iteration + const contentTypes = contentTypesMapperDb.data?.ContentTypesMappers || []; + contentTypes.forEach((ct: any) => { + if (ct?.otherCmsUid) { + existingContentTypes.add(ct?.otherCmsUid); + } + }); + } catch (error) { + console.warn(`Failed to read iteration ${i}: `, error); + // Continue checking other iterations + continue; + } + } + + return Array.from(existingContentTypes); +}; + +/** + * Checks if a content type should be skipped during creation. + * This is a convenience wrapper around isContentTypeAlreadyCreated. + * + * @param projectId - The project ID to check + * @param contentTypeUid - The content type UID to check + * @param currentIteration - The current iteration number + * @returns true if content type should be skipped, false if it should be created + */ +export const shouldSkipContentTypeCreation = async ( + projectId: string, + contentTypeUid: string, + currentIteration: number +): Promise => { + return await isContentTypeAlreadyCreated(projectId, contentTypeUid, currentIteration) +}; \ No newline at end of file diff --git a/api/src/utils/entry-duplicate.utils.ts b/api/src/utils/entry-duplicate.utils.ts new file mode 100644 index 000000000..55e78502b --- /dev/null +++ b/api/src/utils/entry-duplicate.utils.ts @@ -0,0 +1,30 @@ +import getEntryMapperDb from "../models/EntryMapper.js"; +import ProjectModelLowdb from "../models/project-lowdb.js"; + +export const isDuplicateEntry = async (projectId: string) => { + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + const entryMapper = getEntryMapperDb(projectId, iteration); + await entryMapper.read(); + // const entryMapperData = entryMapper.chain.get("entry_mapper").value(); + const seen = new Map(); + + await entryMapper.update((data: any) => { + data?.entry_mapper?.forEach((item: any, index: number) => { + const key = `${item?.contentTypeId}_${item?.language}_${item?.entryName}`; + + if (seen.has(key)) { + const firstIndex = seen.get(key); + data.entry_mapper[firstIndex].isDuplicateEntry = true; + item.isDuplicateEntry = true; + } + else{ + seen.set(key, index); + } + }); + }); +}; diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs new file mode 100644 index 000000000..29f667772 --- /dev/null +++ b/api/src/utils/entry-update-script.cjs @@ -0,0 +1,184 @@ +"use strict"; + +const isAssetField = (value) => + value && typeof value === 'object' && !Array.isArray(value) && + 'urlPath' in value && 'filename' in value; + +/** Export JSON metadata — not Contentstack content-type field UIDs (WordPress entries are flat). */ +const FLAT_PAYLOAD_SKIP = new Set([ + 'uid', + 'publish_details', + 'locale', + 'tags', + 'ACL', + '_version', + 'created_at', + 'updated_at', + 'created_by', + 'updated_by', + '_content_type_uid', + 'content', +]); + +/** + * 3-way asset resolution: + * 1. newMapping has a UID for this source asset → asset was re-imported, always use it + * 2. oldMapping has a UID but stack differs → user changed it manually, keep user's choice + * 3. oldMapping has a UID and stack matches → nothing changed, keep as-is + * 4. No mapping at all → keep whatever is on the stack + */ +const resolveAssetField = (fieldName, entryUid, updateValue, stackValue, oldMapping, newMapping) => { + const sourceCmsUid = updateValue?.uid; + if (!sourceCmsUid) { + return stackValue || updateValue; + } + + const newMappedUid = newMapping[sourceCmsUid]; + const oldMappedUid = oldMapping[sourceCmsUid]; + const currentStackUid = stackValue?.uid; + + if (newMappedUid) { + console.info(`[${entryUid}] "${fieldName}": Using newly imported asset "${newMappedUid}" (source: ${sourceCmsUid})`); + if (stackValue && typeof stackValue === 'object') { + return { ...stackValue, uid: newMappedUid }; + } + return { uid: newMappedUid }; + } + + if (oldMappedUid && currentStackUid && currentStackUid !== oldMappedUid) { + console.info(`[${entryUid}] "${fieldName}": Keeping user-modified asset "${currentStackUid}" (original was: ${oldMappedUid})`); + return stackValue; + } + + if (oldMappedUid) { + console.info(`[${entryUid}] "${fieldName}": Keeping original mapped asset "${oldMappedUid}"`); + if (stackValue && typeof stackValue === 'object') { + return stackValue; + } + return { ...updateValue, uid: oldMappedUid }; + } + + console.info(`[${entryUid}] "${fieldName}": No mapping found, keeping stack asset`); + return stackValue || updateValue; +}; + +/** + * WordPress (and similar) write migration JSON with fields at the root (email, url, …). + * Fetched stack entries keep custom fields under entry.content — merge flat updateData there. + */ +const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping) => { + for (const field of Object.keys(updateData)) { + if (FLAT_PAYLOAD_SKIP.has(field)) { + continue; + } + if (field === 'title') { + if (updateData?.title !== undefined && updateData?.title !== null) { + entry.title = updateData?.title; + } + continue; + } + let nextVal = updateData[field]; + if (isAssetField(nextVal)) { + nextVal = resolveAssetField( + field, + entryUid, + nextVal, + entry?.content[field], + oldMapping, + newMapping + ); + } + entry.content[field] = nextVal; + } + await entry.update(); +}; + +module.exports = async ({ + migration, + config, + stackSDKInstance +}) => { + const assetMapping = config.__assetMapping__ || { old: {}, new: {} }; + delete config.__assetMapping__; + + const oldMapping = assetMapping.old || {}; + const newMapping = assetMapping.new || {}; + console.info(`Asset mappings loaded — old: ${Object.keys(oldMapping).length}, new: ${Object.keys(newMapping).length}`); + + const contentTypes = Object.keys(config); + console.info('contentTypes', contentTypes); + + const updateEntryTask = () => { + return { + title: "Update Entries", + successMessage: 'Entries Updated Successfully', + failedMessage: "Failed to update entries", + task: async () => { + try { + for (const contentType of contentTypes) { + const entryUids = Object.keys(config[contentType]); + console.info(`Processing content type: ${contentType}, entries: ${entryUids.length}`); + + for (const entryUid of entryUids) { + const entryRef = stackSDKInstance + .contentType(contentType) + .entry(entryUid); + + + const entry = await entryRef?.fetch(); + const updateData = JSON.parse(JSON.stringify(config[contentType][entryUid])); + + const hasStackContent = entry?.content && typeof entry?.content === 'object'; + const hasNestedUpdate = updateData?.content && typeof updateData?.content === 'object'; + + if (hasStackContent && hasNestedUpdate) { + for (const field of Object.keys(updateData?.content)) { + if (isAssetField(updateData?.content[field])) { + updateData.content[field] = resolveAssetField( + field, + entryUid, + updateData?.content[field], + entry?.content[field], + oldMapping, + newMapping + ); + } + } + Object.assign(entry?.content, updateData?.content); + await entry.update(); + } else if (hasStackContent) { + console.info(`[${entryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)`); + await mergeFlatPayloadIntoEntry(entry, entryUid, updateData, oldMapping, newMapping); + } else { + if (updateData && entry) { + for (const field of Object.keys(updateData)) { + if (isAssetField(updateData[field])) { + console.info('field is asset field'); + updateData[field] = resolveAssetField( + field, + entryUid, + updateData[field], + entry[field], + oldMapping, + newMapping + ); + } + } + } + Object.assign(entry, updateData); + await entry.update(); + } + console.info(`Updated entry: ${entryUid}`); + } + } + console.info('All entries updated successfully'); + } catch (error) { + console.error(error); + throw error; + } + }, + }; + }; + + migration.addTask(updateEntryTask()); +}; diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts new file mode 100644 index 000000000..f4dd394c1 --- /dev/null +++ b/api/src/utils/entry-update.utils.ts @@ -0,0 +1,172 @@ +import getEntryMapperDb from "../models/EntryMapper.js"; +import ProjectModelLowdb from "../models/project-lowdb.js"; +import path from "path"; +import fs from "node:fs"; +import { MIGRATION_DATA_CONFIG } from "../constants/index.js"; + +/** + * Helper function to write log entries to file + */ +const writeLogEntry = (message: string, methodName: string, loggerPath?: string) => { + if (loggerPath) { + const directLogEntry = { + level: 'info', + message, + methodName, + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(loggerPath, JSON.stringify(directLogEntry) + '\n'); + } +}; + +export const removeEntriesFromDatabase = async (projectId: string, loggerPath?: string): Promise => { + const entriesToUpdate: Record> = {}; + + await ProjectModelLowdb.read(); + const projectData = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + const iteration = projectData?.iteration || 1; + const stackId = projectData?.destination_stack_id; + const updateEntryDataDb = getEntryMapperDb(projectId, iteration); + await updateEntryDataDb.read(); + + const entryMapperItems = updateEntryDataDb.chain.get("entry_mapper").value(); + if (!entryMapperItems?.length || !stackId) { + writeLogEntry("No entry mapper items found or stackId missing, skipping removal.", "removeEntriesFromDatabase", loggerPath); + return null; + } + + const sitecoreUids = new Set( + entryMapperItems.map((item: { otherCmsEntryUid: string }) => item?.otherCmsEntryUid) + ); + + const updateUidMap = new Map(); + for (const item of entryMapperItems) { + if (item.isUpdate) { + updateUidMap.set(item?.otherCmsEntryUid, item?.contentstackEntryUid); + } + } + + const entriesDir = path.join( + process.cwd(), + MIGRATION_DATA_CONFIG.DATA, + stackId, + MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME + ); + + if (!fs.existsSync(entriesDir)) { + writeLogEntry(`Entries directory not found: ${entriesDir}`, "removeEntriesFromDatabase", loggerPath); + return null; + } + + const contentTypeDirs = fs.readdirSync(entriesDir, { withFileTypes: true }) + .filter((dirent) => dirent.isDirectory()); + + for (const ctDir of contentTypeDirs) { + const contentTypeName = ctDir.name; + const ctPath = path.join(entriesDir, contentTypeName); + const localeDirs = fs.readdirSync(ctPath, { withFileTypes: true }) + ?.filter((dirent) => dirent?.isDirectory()); + + for (const localeDir of localeDirs) { + const localePath = path.join(ctPath, localeDir.name); + const jsonFiles = fs.readdirSync(localePath) + ?.filter((file) => file?.endsWith(".json") && file !== "index.json"); + + for (const jsonFile of jsonFiles) { + const filePath = path.join(localePath, jsonFile); + const raw = fs.readFileSync(filePath, "utf-8"); + const data = JSON.parse(raw); + + let modified = false; + for (const key of Object?.keys(data)) { + if (sitecoreUids.has(key)) { + const csEntryUid = updateUidMap.get(key); + if (csEntryUid) { + const entryData = { ...data[key] }; + delete entryData?.uid; + + if (!entriesToUpdate[contentTypeName]) { + entriesToUpdate[contentTypeName] = {}; + } + entriesToUpdate[contentTypeName][csEntryUid] = entryData; + writeLogEntry(`Collected update entry "${csEntryUid}" for content type "${contentTypeName}"`, "removeEntriesFromDatabase", loggerPath); + writeLogEntry(`Entry "${key}" has been prepared for update in Contentstack as "${csEntryUid}"`, "removeEntriesFromDatabase", loggerPath); + } + + delete data[key]; + modified = true; + writeLogEntry(`Removed entry "${key}" from ${filePath}`, "removeEntriesFromDatabase", loggerPath); + writeLogEntry(`Entry "${key}" has been removed from migration data (will be updated instead of created)`, "removeEntriesFromDatabase", loggerPath); + } + } + + if (modified) { + fs.writeFileSync(filePath, JSON.stringify(data), "utf-8"); + } + } + } + } + + const configDir = path.join(process.cwd(), "database", projectId, iteration.toString()); + fs.mkdirSync(configDir, { recursive: true }); + const configPath = path.join(configDir, "updated-entries.json"); + fs.writeFileSync(configPath, JSON.stringify(entriesToUpdate), "utf-8"); + + writeLogEntry("Finished removing entries from cmsMigrationData.", "removeEntriesFromDatabase", loggerPath); + writeLogEntry(`Config written to: ${configPath}`, "removeEntriesFromDatabase", loggerPath); + writeLogEntry(`Total entries prepared for update: ${Object?.keys(entriesToUpdate)?.reduce((total, ct) => total + Object?.keys(entriesToUpdate[ct])?.length, 0)}`, "removeEntriesFromDatabase", loggerPath); + return configPath; +}; + +/** + * Reads old (previous iteration) and new (current iteration) asset uid mappings + * and merges them into the updated-entries config file under __assetMapping__. + * This allows the entry-update-script to resolve asset references using a 3-way comparison: + * - newMapping: asset just re-imported in this iteration → always wins + * - oldMapping vs stack: detect if user manually changed the asset + */ +export const enrichConfigWithAssetMapping = ( + configFilePath: string, + projectId: string, + iteration: number, + loggerPath?: string +): void => { + const dbBase = path.join(process.cwd(), "database", projectId); + + let oldAssetMapping: Record = {}; + if (iteration > 1) { + const oldPath = path.join(dbBase, (iteration - 1).toString(), "uid-mapper.json"); + if (fs.existsSync(oldPath)) { + try { + const data = JSON.parse(fs.readFileSync(oldPath, "utf-8")); + oldAssetMapping = data?.assets || {}; + writeLogEntry(`Loaded ${Object.keys(oldAssetMapping).length} old asset mappings from iteration ${iteration - 1}`, "enrichConfigWithAssetMapping", loggerPath); + } catch (err) { + console.error("Failed to read old uid-mapper:", err); + } + } else { + writeLogEntry(`No old asset mapping found for iteration ${iteration - 1}`, "enrichConfigWithAssetMapping", loggerPath); + } + } + + let newAssetMapping: Record = {}; + const newPath = path.join(dbBase, iteration.toString(), "uid-mapper.json"); + if (fs.existsSync(newPath)) { + try { + const data = JSON.parse(fs.readFileSync(newPath, "utf-8")); + newAssetMapping = data?.assets || {}; + writeLogEntry(`Loaded ${Object.keys(newAssetMapping).length} new asset mappings from iteration ${iteration}`, "enrichConfigWithAssetMapping", loggerPath); + } catch (err) { + console.error("Failed to read new uid-mapper:", err); + } + } else { + writeLogEntry(`No new asset mapping found for iteration ${iteration}`, "enrichConfigWithAssetMapping", loggerPath); + } + + writeLogEntry(`Asset mapping enriched into config: old=${Object?.keys(oldAssetMapping)?.length} keys, new=${Object?.keys(newAssetMapping)?.length} keys`, "enrichConfigWithAssetMapping", loggerPath); + writeLogEntry(`Asset mapping configuration has been enriched for iteration ${iteration}`, "enrichConfigWithAssetMapping", loggerPath); + writeLogEntry(`Asset references will be resolved using combined old and new mappings`, "enrichConfigWithAssetMapping", loggerPath); +}; \ No newline at end of file diff --git a/api/src/utils/field-attacher.utils.ts b/api/src/utils/field-attacher.utils.ts index 532f69ac9..cd77dc392 100644 --- a/api/src/utils/field-attacher.utils.ts +++ b/api/src/utils/field-attacher.utils.ts @@ -1,14 +1,23 @@ import ProjectModelLowdb from "../models/project-lowdb.js"; -import ContentTypesMapperModelLowdb from "../models/contentTypesMapper-lowdb.js"; -import FieldMapperModel from "../models/FieldMapper.js"; +import getContentTypesMapperDb from "../models/contentTypesMapper-lowdb.js"; +import getFieldMapperDb from "../models/FieldMapper.js"; import { contenTypeMaker } from "./content-type-creator.utils.js"; +import { shouldSkipContentTypeCreation } from "./content-type-checker.utils.js"; +import { sanitizeProjectId } from "./sanitize-path.utils.js"; export const fieldAttacher = async ({ projectId, orgId, destinationStackId, region, user_id, is_sso }: any) => { + const safeProjectId = sanitizeProjectId(projectId); + if (!safeProjectId) { + throw new Error("Invalid project identifier"); + } await ProjectModelLowdb.read(); const projectData: any = ProjectModelLowdb.chain.get("projects").find({ - id: projectId, + id: safeProjectId, org_id: orgId, }).value() + const iteration = projectData?.iteration || 1; + const ContentTypesMapperModelLowdb = getContentTypesMapperDb(safeProjectId, iteration); + const FieldMapperModel = getFieldMapperDb(safeProjectId, iteration); await ContentTypesMapperModelLowdb.read(); await FieldMapperModel.read(); const contentTypes = []; @@ -16,18 +25,31 @@ export const fieldAttacher = async ({ projectId, orgId, destinationStackId, regi for await (const contentId of projectData?.content_mapper ?? []) { const contentType: any = ContentTypesMapperModelLowdb.chain .get("ContentTypesMappers") - .find({ id: contentId, projectId: projectId }) + .find({ id: contentId, projectId: safeProjectId }) .value(); if (contentType?.fieldMapping?.length) { contentType.fieldMapping = contentType?.fieldMapping?.map((fieldUid: any) => { const field = FieldMapperModel.chain .get("field_mapper") - .find({ id: fieldUid, contentTypeId: contentId, projectId: projectId }) + .find({ id: fieldUid, contentTypeId: contentId, projectId: safeProjectId }) .value() return field; }) } - await contenTypeMaker({ contentType, destinationStackId, projectId, newStack: projectData?.stackDetails?.isNewStack, keyMapper: projectData?.mapperKeys, region, user_id, is_sso }) + + if (iteration === 1) { + await contenTypeMaker({ contentType, destinationStackId, projectId: safeProjectId, newStack: projectData?.stackDetails?.isNewStack, keyMapper: projectData?.mapperKeys, region, user_id, is_sso }) + + } + else { + const shouldSkip = await shouldSkipContentTypeCreation(safeProjectId, contentType?.otherCmsUid, iteration); + if (!shouldSkip) { + console.info(`Creating new content type: ${contentType.otherCmsUid}`); + await contenTypeMaker({ contentType, destinationStackId, projectId: safeProjectId, newStack: projectData?.stackDetails?.isNewStack, keyMapper: projectData?.mapperKeys, region, user_id, is_sso }) + } else { + console.info(`Skipping content type creation: ${contentType.otherCmsUid} (already exists from previous iteration)`); + } + } contentTypes?.push?.(contentType); } } diff --git a/api/src/utils/package.json b/api/src/utils/package.json new file mode 100644 index 000000000..cb27b8a55 --- /dev/null +++ b/api/src/utils/package.json @@ -0,0 +1,10 @@ +{ + "name": "MigrationPackage", + "version": "1.0.0", + "main": "", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "description": "" + } \ No newline at end of file diff --git a/api/src/utils/sanitize-path.utils.ts b/api/src/utils/sanitize-path.utils.ts index 961bf3eff..45f9a16dc 100644 --- a/api/src/utils/sanitize-path.utils.ts +++ b/api/src/utils/sanitize-path.utils.ts @@ -73,6 +73,26 @@ export const sanitizeStackId = ( return safeValue; }; +/** Same rules as stack IDs (UUIDs, API keys); use for path segments such as `database//`. */ +export const sanitizeProjectId = sanitizeStackId; + +/** + * Throws if {@link targetPath} resolves outside {@link baseDir} (after path.resolve). + */ +export const assertResolvedPathUnderBase = ( + baseDir: string, + targetPath: string +): void => { + const base = path.resolve(baseDir); + const resolved = path.resolve(targetPath); + const rel = path.relative(base, resolved); + if (rel.startsWith('..') || path.isAbsolute(rel)) { + throw new Error( + 'Invalid path: resolved location is outside the allowed base directory' + ); + } +}; + /** * Resolves and validates a safe path dynamically. * Supports full paths, path.join(), and path.resolve(). diff --git a/api/tests/unit/helper/index.test.ts b/api/tests/unit/helper/index.test.ts new file mode 100644 index 000000000..dad32ffb7 --- /dev/null +++ b/api/tests/unit/helper/index.test.ts @@ -0,0 +1,158 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +const mockCreateConnection = vi.fn(); +const mockConnect = vi.fn(); +const mockDestroy = vi.fn(); +const mockEnd = vi.fn(); +const mockCustomLogger = vi.fn().mockResolvedValue(undefined); + +vi.mock('mysql2', () => ({ + default: { + createConnection: (...args: unknown[]) => mockCreateConnection(...args), + }, +})); + +vi.mock('../../../src/utils/custom-logger.utils.js', () => ({ + default: (...args: unknown[]) => mockCustomLogger(...args), +})); + +import { createDbConnection, getDbConnection } from '../../../src/helper/index.js'; + +describe('helper createDbConnection', () => { + const config = { + host: 'h', + user: 'u', + password: 'p', + database: 'd', + port: '3306', + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockCustomLogger.mockReset(); + mockCustomLogger.mockResolvedValue(undefined); + mockDestroy.mockClear(); + mockEnd.mockImplementation((cb?: (err?: Error) => void) => { + if (cb) cb(); + }); + mockCreateConnection.mockReturnValue({ + connect: mockConnect, + destroy: mockDestroy, + end: mockEnd, + }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('resolves connection on successful connect', async () => { + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(null); + }); + const conn = await createDbConnection(config, 'proj', 'stack'); + expect(conn).toBeDefined(); + expect(mockCreateConnection).toHaveBeenCalled(); + }); + + it('still resolves connection when info logger throws synchronously', async () => { + mockCustomLogger.mockImplementation((_p, _s, level: string) => { + if (level === 'info') throw new Error('sync log'); + return Promise.resolve(); + }); + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(null); + }); + const conn = await createDbConnection(config, 'proj', 'stack'); + expect(conn).toBeDefined(); + }); + + it('still resolves connection when info logger rejects asynchronously', async () => { + mockCustomLogger.mockImplementation((_p, _s, level: string) => + level === 'info' ? Promise.reject(new Error('async log')) : Promise.resolve() + ); + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(null); + }); + const conn = await createDbConnection(config, 'proj', 'stack'); + expect(conn).toBeDefined(); + }); + + it('rejects when connect returns error', async () => { + const dbErr = new Error('conn refused'); + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(dbErr); + }); + await expect(createDbConnection(config, 'proj', 'stack')).rejects.toThrow('conn refused'); + expect(mockEnd).toHaveBeenCalled(); + }); + + it('rejects with original DB error when error logger throws synchronously', async () => { + const dbErr = new Error('conn refused'); + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(dbErr); + }); + mockCustomLogger.mockImplementation((_p, _s, level: string) => { + if (level === 'error') throw new Error('logger broke'); + return Promise.resolve(); + }); + await expect(createDbConnection(config, 'proj', 'stack')).rejects.toThrow('conn refused'); + }); + + it('logs warn when connection.end reports an error after connect failure', async () => { + const dbErr = new Error('conn refused'); + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(dbErr); + }); + mockEnd.mockImplementation((cb?: (e?: Error) => void) => { + if (cb) cb(new Error('end failed')); + }); + await expect(createDbConnection(config, 'proj', 'stack')).rejects.toThrow('conn refused'); + }); + + it('returns null when createConnection throws synchronously', async () => { + mockCreateConnection.mockImplementation(() => { + throw new Error('bad config'); + }); + const conn = await createDbConnection(config, 'proj', 'stack'); + expect(conn).toBeNull(); + }); + + it('times out when connect never completes', async () => { + vi.useFakeTimers(); + mockConnect.mockImplementation(() => {}); + const p = createDbConnection(config, 'proj', 'stack', 5000); + vi.advanceTimersByTime(5000); + await expect(p).rejects.toThrow('timed out'); + expect(mockDestroy).toHaveBeenCalled(); + }); + + it('destroys connection if callback arrives after timeout', async () => { + vi.useFakeTimers(); + let cbRef: ((err: Error | null) => void) | undefined; + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cbRef = cb; + }); + const p = createDbConnection(config, 'proj', 'stack', 50); + vi.advanceTimersByTime(50); + await expect(p).rejects.toThrow('timed out'); + + cbRef?.(null); + expect(mockDestroy).toHaveBeenCalledTimes(2); + }); + + it('getDbConnection throws when connection is null', async () => { + mockCreateConnection.mockImplementation(() => { + throw new Error('fail'); + }); + await expect(getDbConnection(config, 'p', 's')).rejects.toThrow('Could not establish database connection'); + }); + + it('getDbConnection returns connection on success', async () => { + mockConnect.mockImplementation((cb: (err: Error | null) => void) => { + cb(null); + }); + const conn = await getDbConnection(config, 'p', 's'); + expect(conn).toBe(mockCreateConnection.mock.results[0].value); + }); +}); diff --git a/api/tests/unit/middlewares/auth.middleware.test.ts b/api/tests/unit/middlewares/auth.middleware.test.ts index 248e596a3..513858a6b 100644 --- a/api/tests/unit/middlewares/auth.middleware.test.ts +++ b/api/tests/unit/middlewares/auth.middleware.test.ts @@ -24,7 +24,7 @@ describe('auth.middleware', () => { next = vi.fn(); }); - it('should return 401 when app_token header is missing', () => { + it('should return 401 when app_token header is missing or empty', () => { req.get.mockReturnValue(undefined); authenticateUser(req, res, next); @@ -34,6 +34,12 @@ describe('auth.middleware', () => { expect.objectContaining({ message: 'Unauthorized - Token missing' }) ); expect(next).not.toHaveBeenCalled(); + + vi.clearAllMocks(); + req.get.mockReturnValue(''); + authenticateUser(req, res, next); + expect(res.status).toHaveBeenCalledWith(401); + expect(next).not.toHaveBeenCalled(); }); it('should return 401 when JWT verification fails', () => { diff --git a/api/tests/unit/middlewares/logger.middleware.test.ts b/api/tests/unit/middlewares/logger.middleware.test.ts new file mode 100644 index 000000000..31dd98a91 --- /dev/null +++ b/api/tests/unit/middlewares/logger.middleware.test.ts @@ -0,0 +1,8 @@ +import { describe, it, expect } from 'vitest'; + +describe('logger.middleware', () => { + it('exports express-winston middleware', async () => { + const mod = await import('../../../src/middlewares/logger.middleware.js'); + expect(mod.default).toBeDefined(); + }); +}); diff --git a/api/tests/unit/middlewares/unmatched-routes.middleware.test.ts b/api/tests/unit/middlewares/unmatched-routes.middleware.test.ts index f682aebd3..25ec75e38 100644 --- a/api/tests/unit/middlewares/unmatched-routes.middleware.test.ts +++ b/api/tests/unit/middlewares/unmatched-routes.middleware.test.ts @@ -2,21 +2,19 @@ import { describe, it, expect, vi } from 'vitest'; import { unmatchedRoutesMiddleware } from '../../../src/middlewares/unmatched-routes.middleware.js'; describe('unmatched-routes.middleware', () => { - it('should return 404 with route error message', () => { + it('returns 404 JSON payload', () => { const req = {} as any; - const res = { - status: vi.fn().mockReturnThis(), - json: vi.fn().mockReturnThis(), - }; + const status = vi.fn().mockReturnThis(); + const json = vi.fn(); + const res = { status, json } as any; - unmatchedRoutesMiddleware(req, res as any); + unmatchedRoutesMiddleware(req, res); - expect(res.status).toHaveBeenCalledWith(404); - expect(res.json).toHaveBeenCalledWith({ - error: { - code: 404, - message: 'Sorry, the requested resource is not available.', - }, - }); + expect(status).toHaveBeenCalledWith(404); + expect(json).toHaveBeenCalledWith( + expect.objectContaining({ + error: expect.objectContaining({ code: 404 }), + }) + ); }); }); diff --git a/api/tests/unit/models/FieldMapper.model.test.ts b/api/tests/unit/models/FieldMapper.model.test.ts index 96efda500..f8fe1a060 100644 --- a/api/tests/unit/models/FieldMapper.model.test.ts +++ b/api/tests/unit/models/FieldMapper.model.test.ts @@ -24,7 +24,8 @@ describe('FieldMapper model', () => { }); it('should export db with field_mapper array in default data', async () => { - const fieldMapperDb = (await import('../../../src/models/FieldMapper.js')).default; + const getFieldMapperDb = (await import('../../../src/models/FieldMapper.js')).default; + const fieldMapperDb = getFieldMapperDb('test-project', 0); expect(fieldMapperDb).toBeDefined(); expect(fieldMapperDb.data).toBeDefined(); @@ -34,7 +35,8 @@ describe('FieldMapper model', () => { }); it('should have correct default structure for FieldMapper', async () => { - const fieldMapperDb = (await import('../../../src/models/FieldMapper.js')).default; + const getFieldMapperDb = (await import('../../../src/models/FieldMapper.js')).default; + const fieldMapperDb = getFieldMapperDb('test-project', 0); expect(fieldMapperDb.data).toMatchObject({ field_mapper: [], diff --git a/api/tests/unit/models/contentTypesMapper-lowdb.model.test.ts b/api/tests/unit/models/contentTypesMapper-lowdb.model.test.ts index 66ce8ce16..ab7a5eb29 100644 --- a/api/tests/unit/models/contentTypesMapper-lowdb.model.test.ts +++ b/api/tests/unit/models/contentTypesMapper-lowdb.model.test.ts @@ -24,7 +24,8 @@ describe('contentTypesMapper-lowdb model', () => { }); it('should export db with ContentTypesMappers array in default data', async () => { - const contentTypesDb = (await import('../../../src/models/contentTypesMapper-lowdb.js')).default; + const getContentTypesMapperDb = (await import('../../../src/models/contentTypesMapper-lowdb.js')).default; + const contentTypesDb = getContentTypesMapperDb('test-project', 0); expect(contentTypesDb).toBeDefined(); expect(contentTypesDb.data).toBeDefined(); @@ -34,7 +35,8 @@ describe('contentTypesMapper-lowdb model', () => { }); it('should have correct default structure for ContentTypeMapperDocument', async () => { - const contentTypesDb = (await import('../../../src/models/contentTypesMapper-lowdb.js')).default; + const getContentTypesMapperDb = (await import('../../../src/models/contentTypesMapper-lowdb.js')).default; + const contentTypesDb = getContentTypesMapperDb('test-project', 0); expect(contentTypesDb.data).toMatchObject({ ContentTypesMappers: [], diff --git a/api/tests/unit/models/entry-mapper.model.test.ts b/api/tests/unit/models/entry-mapper.model.test.ts new file mode 100644 index 000000000..de89b87a1 --- /dev/null +++ b/api/tests/unit/models/entry-mapper.model.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockMkdirSync = vi.fn(); + +vi.mock('node:fs', () => ({ + default: { mkdirSync: mockMkdirSync }, +})); + +vi.mock('lowdb/node', () => ({ + JSONFile: vi.fn(function JSONFile(this: { path: unknown }, p: unknown) { + this.path = p; + }), +})); + +vi.mock('../../../src/utils/lowdb-lodash.utils.js', () => ({ + default: class LowWithLodash { + adapter: unknown; + data: unknown; + constructor(adapter: unknown, data: unknown) { + this.adapter = adapter; + this.data = data; + } + }, +})); + +describe('EntryMapper factory', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('getEntryMapperDb creates db under database/{projectId}/{iteration}', async () => { + const getEntryMapperDb = (await import('../../../src/models/EntryMapper.js')).default; + const db = getEntryMapperDb('proj-a', 3); + expect(mockMkdirSync).toHaveBeenCalled(); + expect(db).toBeDefined(); + }); +}); diff --git a/api/tests/unit/models/uid-mapper.model.test.ts b/api/tests/unit/models/uid-mapper.model.test.ts new file mode 100644 index 000000000..25e9ac074 --- /dev/null +++ b/api/tests/unit/models/uid-mapper.model.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockMkdirSync = vi.fn(); + +vi.mock('node:fs', () => ({ + default: { mkdirSync: mockMkdirSync }, +})); + +vi.mock('lowdb/node', () => ({ + JSONFile: vi.fn(function JSONFile(this: { path: unknown }, p: unknown) { + this.path = p; + }), +})); + +vi.mock('../../../src/utils/lowdb-lodash.utils.js', () => ({ + default: class LowWithLodash { + adapter: unknown; + data: unknown; + constructor(adapter: unknown, data: unknown) { + this.adapter = adapter; + this.data = data; + } + }, +})); + +describe('uidMapper factory', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('getUidMapperDb creates db under database/{projectId}/{iteration}', async () => { + const getUidMapperDb = (await import('../../../src/models/uidMapper.js')).default; + const db = getUidMapperDb('proj-b', 2); + expect(mockMkdirSync).toHaveBeenCalled(); + expect(db).toBeDefined(); + }); +}); diff --git a/api/tests/unit/routes/contentMapper.routes.test.ts b/api/tests/unit/routes/contentMapper.routes.test.ts index 5b9474f8b..b949cb1ab 100644 --- a/api/tests/unit/routes/contentMapper.routes.test.ts +++ b/api/tests/unit/routes/contentMapper.routes.test.ts @@ -12,6 +12,8 @@ vi.mock('../../../src/controllers/projects.contentMapper.controller.js', () => ( resetContentType: vi.fn((_req: any, res: any) => res.status(200).json({})), removeContentMapper: vi.fn((_req: any, res: any) => res.status(200).json({})), updateContentMapper: vi.fn((_req: any, res: any) => res.status(200).json({})), + getEntryMapping: vi.fn((_req: any, res: any) => res.status(200).json({})), + updateEntryStatus: vi.fn((_req: any, res: any) => res.status(200).json({})), }, })); diff --git a/api/tests/unit/routes/migration.routes.test.ts b/api/tests/unit/routes/migration.routes.test.ts index 85ce2f391..2a6b1d7f7 100644 --- a/api/tests/unit/routes/migration.routes.test.ts +++ b/api/tests/unit/routes/migration.routes.test.ts @@ -10,6 +10,7 @@ vi.mock('../../../src/controllers/migration.controller.js', () => ({ getAuditData: vi.fn((_req: any, res: any) => res.status(200).json({})), saveLocales: vi.fn((_req: any, res: any) => res.status(200).json({})), saveMappedLocales: vi.fn((_req: any, res: any) => res.status(200).json({})), + restartMigration: vi.fn((_req: any, res: any) => res.status(200).json({})), }, })); diff --git a/api/tests/unit/services/auth.service.refresh-oauth.test.ts b/api/tests/unit/services/auth.service.refresh-oauth.test.ts new file mode 100644 index 000000000..40382aced --- /dev/null +++ b/api/tests/unit/services/auth.service.refresh-oauth.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { + mockAuthRead, + mockAuthUpdate, + mockChainGet, + mockFindValue, + mockExistsSync, + mockReadFileSync, + mockAxiosPost, +} = vi.hoisted(() => { + const mockFindValue = vi.fn(); + const mockChainGet = vi.fn(() => ({ + find: vi.fn().mockReturnValue({ value: mockFindValue }), + })); + return { + mockAuthRead: vi.fn(), + mockAuthUpdate: vi.fn(), + mockChainGet, + mockFindValue, + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockAxiosPost: vi.fn(), + }; +}); + +vi.mock('axios', () => ({ + default: { post: (...args: unknown[]) => mockAxiosPost(...args) }, +})); + +vi.mock('fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + }, +})); + +vi.mock('../../../src/utils/logger.js', () => ({ + default: { error: vi.fn(), info: vi.fn(), warn: vi.fn() }, +})); + +vi.mock('../../../src/utils/crypto.utils.js', () => ({ + decryptAppConfig: (c: Record) => c, +})); + +vi.mock('../../../src/models/authentication.js', () => ({ + default: { + read: mockAuthRead, + update: mockAuthUpdate, + chain: { + get: mockChainGet, + }, + }, +})); + +import { refreshOAuthToken } from '../../../src/services/auth.service.js'; + +describe('auth.service refreshOAuthToken', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockAuthRead.mockResolvedValue(undefined); + mockAuthUpdate.mockImplementation((fn: (d: { users: unknown[] }) => void) => { + fn({ users: [{ user_id: 'u1', refresh_token: 'rt', email: 'a@b.com', region: 'NA' }] }); + }); + mockFindValue.mockReturnValue({ + user_id: 'u1', + refresh_token: 'rt-old', + email: 'a@b.com', + region: 'NA', + }); + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue( + JSON.stringify({ + oauthData: { + client_id: 'cid', + client_secret: 'sec', + redirect_uri: 'https://cb', + }, + }) + ); + mockAxiosPost.mockResolvedValue({ + data: { access_token: 'new-at', refresh_token: 'new-rt' }, + }); + }); + + it('throws when user record missing', async () => { + mockFindValue.mockReturnValue(null); + await expect(refreshOAuthToken('u1')).rejects.toThrow('User record not found'); + }); + + it('throws when refresh_token missing on user', async () => { + mockFindValue.mockReturnValue({ user_id: 'u1', email: 'a@b.com' }); + await expect(refreshOAuthToken('u1')).rejects.toThrow('No refresh token available'); + }); + + it('throws when app.json missing', async () => { + mockExistsSync.mockReturnValue(false); + await expect(refreshOAuthToken('u1')).rejects.toThrow('app.json file not found'); + }); + + it('throws when OAuth client fields missing', async () => { + mockReadFileSync.mockReturnValue(JSON.stringify({ oauthData: {} })); + await expect(refreshOAuthToken('u1')).rejects.toThrow('client_id or client_secret'); + }); + + it('posts refresh request and returns new access_token', async () => { + const token = await refreshOAuthToken('u1'); + expect(token).toBe('new-at'); + expect(mockAxiosPost).toHaveBeenCalled(); + expect(mockAuthUpdate).toHaveBeenCalled(); + }); + + it('wraps axios failures with friendly error', async () => { + mockAxiosPost.mockRejectedValue(new Error('network')); + await expect(refreshOAuthToken('u1')).rejects.toThrow('Failed to refresh token'); + }); +}); diff --git a/api/tests/unit/services/auth.service.sso.test.ts b/api/tests/unit/services/auth.service.sso.test.ts new file mode 100644 index 000000000..a9b7904db --- /dev/null +++ b/api/tests/unit/services/auth.service.sso.test.ts @@ -0,0 +1,237 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const ORG_MISMATCH_GENERIC = + 'Organization mismatch: the authorized org does not match the Migration Tool SSO configuration.'; + +const { + mockGenerateToken, + mockAuthRead, + mockAuthUpdate, + mockFindIndexInner, + mockFindInner, + mockExistsSync, + mockReadFileSync, + mockGetAppOrgUid, + mockGetAppOrganization, + mockChainGet, +} = vi.hoisted(() => { + const mockFindIndexInner = vi.fn(); + const mockFindInner = vi.fn(); + const mockChainGet = vi.fn(() => ({ + findIndex: vi.fn().mockReturnValue({ value: mockFindIndexInner }), + find: vi.fn().mockReturnValue({ value: mockFindInner }), + })); + return { + mockGenerateToken: vi.fn(() => 'app-jwt'), + mockAuthRead: vi.fn(), + mockAuthUpdate: vi.fn(), + mockFindIndexInner, + mockFindInner, + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockGetAppOrgUid: vi.fn(() => 'org-match'), + mockGetAppOrganization: vi.fn(() => { + throw new Error('App organization metadata unavailable in test'); + }), + mockChainGet, + }; +}); + +vi.mock('../../../src/utils/jwt.utils.js', () => ({ + generateToken: (...args: unknown[]) => mockGenerateToken(...args), +})); + +vi.mock('../../../src/models/authentication.js', () => ({ + default: { + read: mockAuthRead, + update: mockAuthUpdate, + chain: { + get: mockChainGet, + }, + }, +})); + +vi.mock('../../../src/utils/logger.js', () => ({ + default: { error: vi.fn(), info: vi.fn(), warn: vi.fn() }, +})); + +vi.mock('fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + }, + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, +})); + +vi.mock('../../../src/utils/crypto.utils.js', () => ({ + decryptAppConfig: (c: Record) => c, +})); + +vi.mock('../../../src/utils/auth.utils.js', () => ({ + getAppOrganizationUID: () => mockGetAppOrgUid(), + getAppOrganization: () => mockGetAppOrganization(), +})); + +import { getAppData, checkSSOAuthStatus } from '../../../src/services/auth.service.js'; +import { authService } from '../../../src/services/auth.service.js'; + +describe('auth.service SSO helpers', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetAppOrganization.mockImplementation(() => { + throw new Error('App organization metadata unavailable in test'); + }); + mockAuthRead.mockResolvedValue(undefined); + mockAuthUpdate.mockImplementation(async (fn: (d: { users: unknown[] }) => void) => { + fn({ users: [] }); + }); + mockFindIndexInner.mockReturnValue(-1); + mockFindInner.mockReturnValue(null); + }); + + describe('getAppData', () => { + it('throws when app.json is missing', async () => { + mockExistsSync.mockReturnValue(false); + await expect(getAppData()).rejects.toThrow('app.json file not found'); + }); + + it('throws when isDefault is true', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify({ isDefault: true })); + await expect(getAppData()).rejects.toThrow('SSO is not configured'); + }); + + it('returns config when valid', async () => { + mockExistsSync.mockReturnValue(true); + const cfg = { isDefault: false, oauthData: { client_id: 'c' } }; + mockReadFileSync.mockReturnValue(JSON.stringify(cfg)); + await expect(getAppData()).resolves.toMatchObject(cfg); + }); + + it('throws on invalid JSON', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue('{ not json'); + await expect(getAppData()).rejects.toThrow('Invalid JSON format'); + }); + }); + + describe('checkSSOAuthStatus', () => { + it('returns not authenticated when user missing', async () => { + mockFindInner.mockReturnValue(null); + await expect(checkSSOAuthStatus('u1')).resolves.toEqual({ + authenticated: false, + message: 'SSO authentication not completed', + }); + }); + + it('returns not authenticated when no access_token', async () => { + mockFindInner.mockReturnValue({ user_id: 'u1', email: 'a@b.com' }); + await expect(checkSSOAuthStatus('u1')).resolves.toMatchObject({ + authenticated: false, + }); + }); + + it('returns not authenticated when org mismatch', async () => { + mockGetAppOrgUid.mockReturnValueOnce('org-match'); + mockFindInner.mockReturnValue({ + user_id: 'u1', + access_token: 'tok', + organization_uid: 'other-org', + region: 'NA', + email: 'a@b.com', + updated_at: new Date().toISOString(), + }); + await expect(checkSSOAuthStatus('u1')).resolves.toMatchObject({ + authenticated: false, + message: ORG_MISMATCH_GENERIC, + }); + }); + + it('returns not authenticated with Contentstack org hint when app org name resolves', async () => { + mockGetAppOrgUid.mockReturnValueOnce('org-match'); + mockGetAppOrganization.mockReturnValueOnce({ name: 'Configured Contentstack Org' }); + mockFindInner.mockReturnValue({ + user_id: 'u1', + access_token: 'tok', + organization_uid: 'other-org', + region: 'NA', + email: 'a@b.com', + updated_at: new Date().toISOString(), + }); + await expect(checkSSOAuthStatus('u1')).resolves.toMatchObject({ + authenticated: false, + message: + 'Organization mismatch: authorize "Configured Contentstack Org" in Contentstack (same org as SSO setup), then try again.', + }); + }); + + it('returns not authenticated when organization_uid is missing', async () => { + mockFindInner.mockReturnValue({ + user_id: 'u1', + access_token: 'tok', + region: 'NA', + email: 'a@b.com', + updated_at: new Date().toISOString(), + }); + await expect(checkSSOAuthStatus('u1')).resolves.toEqual({ + authenticated: false, + message: 'Organization not linked to user', + }); + }); + + it('returns not authenticated when SSO token is older than 10 minutes', async () => { + mockGetAppOrgUid.mockReturnValue('org-match'); + const stale = new Date(Date.now() - 11 * 60 * 1000).toISOString(); + mockFindInner.mockReturnValue({ + user_id: 'u1', + access_token: 'tok', + organization_uid: 'org-match', + region: 'NA', + email: 'a@b.com', + updated_at: stale, + }); + await expect(checkSSOAuthStatus('u1')).resolves.toEqual({ + authenticated: false, + message: 'SSO authentication expired', + }); + }); + + it('returns success when token fresh and org matches', async () => { + mockGetAppOrgUid.mockReturnValue('org-match'); + mockFindInner.mockReturnValue({ + user_id: 'u1', + access_token: 'tok', + organization_uid: 'org-match', + region: 'NA', + email: 'a@b.com', + updated_at: new Date().toISOString(), + }); + + const out = await checkSSOAuthStatus('u1'); + expect(out.authenticated).toBe(true); + expect(out).toHaveProperty('app_token', 'app-jwt'); + expect(mockGenerateToken).toHaveBeenCalled(); + }); + }); + + describe('logout', () => { + it('throws when email missing', async () => { + await expect(authService.logout({ body: {} } as any)).rejects.toThrow('User not found'); + }); + + it('throws when user not in DB', async () => { + mockFindInner.mockReturnValue(null); + await expect( + authService.logout({ body: { email: 'missing@x.com' } } as any) + ).rejects.toThrow(); + }); + + it('removes user and returns 200', async () => { + mockFindInner.mockReturnValue({ email: 'a@b.com', user_id: 'u1' }); + const r = await authService.logout({ body: { email: 'a@b.com' } } as any); + expect(r.status).toBe(200); + expect(mockAuthUpdate).toHaveBeenCalled(); + }); + }); +}); diff --git a/api/tests/unit/services/contentMapper.service.test.ts b/api/tests/unit/services/contentMapper.service.test.ts index d48502d5f..25b0d1535 100644 --- a/api/tests/unit/services/contentMapper.service.test.ts +++ b/api/tests/unit/services/contentMapper.service.test.ts @@ -14,21 +14,79 @@ const { mockFieldMapperUpdate, mockUuidv4, mockFsPromises, -} = vi.hoisted(() => ({ - mockHttps: vi.fn(), - mockGetAuthToken: vi.fn(), - mockGetProjectUtil: vi.fn(), - mockFetchAllPaginatedData: vi.fn(), - mockProjectRead: vi.fn(), - mockProjectUpdate: vi.fn(), - mockProjectWrite: vi.fn(), - mockContentTypesMapperRead: vi.fn(), - mockContentTypesMapperUpdate: vi.fn(), - mockFieldMapperRead: vi.fn(), - mockFieldMapperUpdate: vi.fn(), - mockUuidv4: vi.fn(() => 'uuid-123'), - mockFsPromises: { lstat: vi.fn(), readFile: vi.fn(), realpath: vi.fn() }, -})); + mockContentTypesDb, + mockFieldDb, + getContentTypesMapperDbMock, + getFieldMapperDbMock, + mockEntryMapperDb, + mockUidMapperDb, + getEntryMapperDbMock, + getUidMapperDbMock, +} = vi.hoisted(() => { + const mockContentTypesMapperRead = vi.fn(); + const mockContentTypesMapperUpdate = vi.fn(); + const mockFieldMapperRead = vi.fn(); + const mockFieldMapperUpdate = vi.fn(); + const mockContentTypesChainGet = vi.fn(); + const mockFieldMapperChainGet = vi.fn(); + const mockContentTypesDb = { + read: mockContentTypesMapperRead, + update: mockContentTypesMapperUpdate, + write: vi.fn(), + chain: { get: mockContentTypesChainGet }, + data: { ContentTypesMappers: [] as unknown[] }, + }; + const mockFieldDb = { + read: mockFieldMapperRead, + update: mockFieldMapperUpdate, + write: vi.fn(), + chain: { get: mockFieldMapperChainGet }, + data: { field_mapper: [] as unknown[] }, + }; + const mockEntryMapperRead = vi.fn(); + const mockEntryMapperUpdate = vi.fn(); + const mockEntryMapperChainGet = vi.fn(); + const mockEntryMapperDb = { + read: mockEntryMapperRead, + update: mockEntryMapperUpdate, + write: vi.fn(), + chain: { get: mockEntryMapperChainGet }, + data: { entry_mapper: [] as unknown[] }, + }; + const mockUidMapperRead = vi.fn(); + const mockUidMapperUpdate = vi.fn(); + const mockUidMapperChainGet = vi.fn(); + const mockUidMapperDb = { + read: mockUidMapperRead, + update: mockUidMapperUpdate, + write: vi.fn(), + chain: { get: mockUidMapperChainGet }, + data: { entry: {} as Record, assets: {} as Record }, + }; + return { + mockHttps: vi.fn(), + mockGetAuthToken: vi.fn(), + mockGetProjectUtil: vi.fn(), + mockFetchAllPaginatedData: vi.fn(), + mockProjectRead: vi.fn(), + mockProjectUpdate: vi.fn(), + mockProjectWrite: vi.fn(), + mockContentTypesMapperRead, + mockContentTypesMapperUpdate, + mockFieldMapperRead, + mockFieldMapperUpdate, + mockUuidv4: vi.fn(() => 'uuid-123'), + mockFsPromises: { lstat: vi.fn(), readFile: vi.fn(), realpath: vi.fn() }, + mockContentTypesDb, + mockFieldDb, + getContentTypesMapperDbMock: vi.fn(() => mockContentTypesDb), + getFieldMapperDbMock: vi.fn(() => mockFieldDb), + mockEntryMapperDb, + mockUidMapperDb, + getEntryMapperDbMock: vi.fn(() => mockEntryMapperDb), + getUidMapperDbMock: vi.fn(() => mockUidMapperDb), + }; +}); vi.mock('../../../src/utils/https.utils.js', () => ({ default: mockHttps })); vi.mock('../../../src/utils/auth.utils.js', () => ({ default: mockGetAuthToken })); @@ -55,42 +113,35 @@ vi.mock('../../../src/models/project-lowdb.js', () => { }; }); -vi.mock('../../../src/models/contentTypesMapper-lowdb.js', () => { - const mockChainGet = vi.fn(); - return { - default: { - read: mockContentTypesMapperRead, - update: mockContentTypesMapperUpdate, - write: vi.fn(), - chain: { get: mockChainGet }, - data: { ContentTypesMappers: [] }, - }, - ContentTypesMapper: {}, - }; -}); +vi.mock('../../../src/models/contentTypesMapper-lowdb.js', () => ({ + default: getContentTypesMapperDbMock, + getContentTypesMapperDb: getContentTypesMapperDbMock, + ContentTypesMapper: {}, +})); -vi.mock('../../../src/models/FieldMapper.js', () => { - const mockChainGet = vi.fn(); +vi.mock('../../../src/models/FieldMapper.js', () => ({ + default: getFieldMapperDbMock, +})); + +vi.mock('../../../src/models/EntryMapper.js', () => ({ + default: getEntryMapperDbMock, +})); + +vi.mock('../../../src/models/uidMapper.js', () => ({ + default: getUidMapperDbMock, +})); + +vi.mock('fs', () => { + const mkdirSync = vi.fn(); return { - default: { - read: mockFieldMapperRead, - update: mockFieldMapperUpdate, - write: vi.fn(), - chain: { get: mockChainGet }, - data: { field_mapper: [] }, - }, + default: { promises: mockFsPromises, mkdirSync }, + mkdirSync, + promises: mockFsPromises, }; }); -vi.mock('fs', () => ({ - default: { promises: mockFsPromises }, - promises: mockFsPromises, -})); - import { contentMapperService } from '../../../src/services/contentMapper.service.js'; import ProjectModelLowdb from '../../../src/models/project-lowdb.js'; -import ContentTypesMapperModelLowdb from '../../../src/models/contentTypesMapper-lowdb.js'; -import FieldMapperModel from '../../../src/models/FieldMapper.js'; const createChain = (opts: { find?: unknown; @@ -109,10 +160,17 @@ const createChain = (opts: { describe('contentMapper.service', () => { beforeEach(() => { vi.clearAllMocks(); + (ProjectModelLowdb.chain.get as ReturnType).mockReset(); + (mockContentTypesDb.chain.get as ReturnType).mockReset(); + (mockFieldDb.chain.get as ReturnType).mockReset(); + (mockEntryMapperDb.chain.get as ReturnType).mockReset(); + (mockUidMapperDb.chain.get as ReturnType).mockReset(); mockGetAuthToken.mockResolvedValue('cs-auth-token'); mockProjectRead.mockResolvedValue(undefined); mockContentTypesMapperRead.mockResolvedValue(undefined); mockFieldMapperRead.mockResolvedValue(undefined); + (mockEntryMapperDb.read as ReturnType).mockResolvedValue(undefined); + (mockUidMapperDb.read as ReturnType).mockResolvedValue(undefined); mockProjectUpdate.mockImplementation(async (fn: (d: any) => void) => { const data = ProjectModelLowdb.data as any; if (!data.projects) data.projects = []; @@ -120,23 +178,35 @@ describe('contentMapper.service', () => { fn(data); }); mockContentTypesMapperUpdate.mockImplementation(async (fn: (d: any) => void) => { - const data = ContentTypesMapperModelLowdb.data as any; + const data = mockContentTypesDb.data as any; if (!data.ContentTypesMappers) data.ContentTypesMappers = []; while (data.ContentTypesMappers.length < 2) data.ContentTypesMappers.push({}); fn(data); }); mockFieldMapperUpdate.mockImplementation(async (fn: (d: any) => void) => { - const data = FieldMapperModel.data as any; + const data = mockFieldDb.data as any; if (!data.field_mapper) data.field_mapper = []; fn(data); }); + (mockEntryMapperDb.update as ReturnType).mockImplementation(async (fn: (d: any) => void) => { + const data = mockEntryMapperDb.data as any; + if (!data.entry_mapper) data.entry_mapper = []; + fn(data); + }); + (mockUidMapperDb.update as ReturnType).mockImplementation(async (fn: (d: any) => void) => { + fn(mockUidMapperDb.data as any); + }); mockFetchAllPaginatedData.mockResolvedValue([]); ProjectModelLowdb.data = { projects: [] }; - ContentTypesMapperModelLowdb.data = { ContentTypesMappers: [] }; - FieldMapperModel.data = { field_mapper: [] }; + mockContentTypesDb.data = { ContentTypesMappers: [] }; + mockFieldDb.data = { field_mapper: [] }; + mockEntryMapperDb.data = { entry_mapper: [] }; + mockUidMapperDb.data = { entry: {}, assets: {} }; (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); - (ContentTypesMapperModelLowdb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); - (FieldMapperModel.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); + (mockContentTypesDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); + (mockFieldDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); + (mockEntryMapperDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); + (mockUidMapperDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); }); describe('putTestData', () => { @@ -171,9 +241,9 @@ describe('contentMapper.service', () => { ProjectModelLowdb.data.projects = [project]; mockProjectWrite.mockResolvedValue(undefined); - (ProjectModelLowdb.chain.get as ReturnType) - .mockReturnValueOnce(createChain({ findIndex: 0 })) - .mockReturnValueOnce(createChain({ find: project })); + (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue( + createChain({ find: project, findIndex: 0 }) + ); const req = { params: { projectId: 'proj-1' }, @@ -209,7 +279,7 @@ describe('contentMapper.service', () => { const contentMapper = { id: 'ct-1', projectId: 'proj-1', otherCmsTitle: 'Blog' }; (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue(createChain({ find: project })); - (ContentTypesMapperModelLowdb.chain.get as ReturnType).mockReturnValue( + (mockContentTypesDb.chain.get as ReturnType).mockReturnValue( createChain({ find: contentMapper }) ); @@ -229,7 +299,7 @@ describe('contentMapper.service', () => { const contentMapper = { id: 'ct-1', projectId: 'proj-1', otherCmsTitle: 'Blog' }; (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue(createChain({ find: project })); - (ContentTypesMapperModelLowdb.chain.get as ReturnType).mockReturnValue( + (mockContentTypesDb.chain.get as ReturnType).mockReturnValue( createChain({ find: contentMapper }) ); @@ -246,7 +316,7 @@ describe('contentMapper.service', () => { describe('getFieldMapping', () => { it('should throw when content type not found', async () => { - (ContentTypesMapperModelLowdb.chain.get as ReturnType).mockReturnValue( + (mockContentTypesDb.chain.get as ReturnType).mockReturnValue( createChain({ find: null }) ); @@ -261,10 +331,10 @@ describe('contentMapper.service', () => { const contentType = { id: 'ct-1', projectId: 'proj-1', fieldMapping: ['f1'] }; const fieldData = { id: 'f1', otherCmsField: 'title', contentstackField: 'title' }; - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValue(createChain({ find: contentType })); - (FieldMapperModel.chain.get as ReturnType) + (mockFieldDb.chain.get as ReturnType) .mockReturnValue(createChain({ find: fieldData })); const req = { @@ -281,9 +351,9 @@ describe('contentMapper.service', () => { const contentType = { id: 'ct-1', projectId: 'proj-1', fieldMapping: ['f1'] }; const fieldData = { id: 'f1', otherCmsField: 'Title', contentstackField: 'title' }; - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValue(createChain({ find: contentType })); - (FieldMapperModel.chain.get as ReturnType) + (mockFieldDb.chain.get as ReturnType) .mockReturnValue(createChain({ find: fieldData })); const req = { @@ -461,9 +531,9 @@ describe('contentMapper.service', () => { it('should return 400 when field has invalid contentstackFieldType', async () => { mockGetProjectUtil.mockResolvedValue(0); ProjectModelLowdb.data.projects = [{ status: 1, current_step: 3 }]; - ContentTypesMapperModelLowdb.data.ContentTypesMappers = [{ id: 'ct-1', status: 1 }]; + mockContentTypesDb.data.ContentTypesMappers = [{ id: 'ct-1', status: 1 }]; - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ findIndex: 0 })) .mockReturnValue(createChain({ find: { id: 'ct-1', projectId: 'proj-1', status: 1 } })); @@ -486,12 +556,12 @@ describe('contentMapper.service', () => { it('should update content type successfully', async () => { mockGetProjectUtil.mockResolvedValue(0); ProjectModelLowdb.data.projects = [{ status: 1, current_step: 3 }]; - ContentTypesMapperModelLowdb.data.ContentTypesMappers = [{ id: 'ct-1', projectId: 'proj-1', status: 1 }]; - FieldMapperModel.data.field_mapper = [ + mockContentTypesDb.data.ContentTypesMappers = [{ id: 'ct-1', projectId: 'proj-1', status: 1 }]; + mockFieldDb.data.field_mapper = [ { id: 'f1', contentTypeId: 'ct-1', contentstackFieldType: 'text', contentstackFieldUid: 'f1' }, ]; - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ findIndex: 0 })) .mockReturnValue(createChain({ find: { id: 'ct-1', projectId: 'proj-1', status: 1 } })); @@ -550,13 +620,13 @@ describe('contentMapper.service', () => { advanced: { initial: {} }, }; - ContentTypesMapperModelLowdb.data.ContentTypesMappers = [contentTypeData]; - FieldMapperModel.data.field_mapper = [fieldData]; + mockContentTypesDb.data.ContentTypesMappers = [contentTypeData]; + mockFieldDb.data.field_mapper = [fieldData]; - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: contentTypeData })) .mockReturnValueOnce(createChain({ findIndex: 0 })); - (FieldMapperModel.chain.get as ReturnType).mockReturnValue(createChain({ find: fieldData })); + (mockFieldDb.chain.get as ReturnType).mockReturnValue(createChain({ find: fieldData })); const req = { params: { orgId: 'org-1', projectId: 'proj-1', contentTypeId: 'ct-1' }, @@ -591,14 +661,14 @@ describe('contentMapper.service', () => { }; const fieldData = { id: 'f1', projectId: 'proj-1', backupFieldType: 'text' }; - ContentTypesMapperModelLowdb.data.ContentTypesMappers = [{ ...contentType, contentstackTitle: 'Old' }]; - FieldMapperModel.data.field_mapper = [fieldData]; + mockContentTypesDb.data.ContentTypesMappers = [{ ...contentType, contentstackTitle: 'Old' }]; + mockFieldDb.data.field_mapper = [fieldData]; (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue(createChain({ find: project })); - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: contentType })) .mockReturnValueOnce(createChain({ findIndex: 0 })); - (FieldMapperModel.chain.get as ReturnType) + (mockFieldDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: fieldData })) .mockReturnValueOnce(createChain({ findIndex: 0 })); @@ -626,10 +696,10 @@ describe('contentMapper.service', () => { (ProjectModelLowdb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: project })) .mockReturnValueOnce(createChain({ findIndex: 0 })); - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: contentType })) .mockReturnValueOnce(createChain({ findIndex: 0 })); - (FieldMapperModel.chain.get as ReturnType).mockReturnValue( + (mockFieldDb.chain.get as ReturnType).mockReturnValue( createChain({ findIndex: 0 }) ); @@ -659,10 +729,10 @@ describe('contentMapper.service', () => { (ProjectModelLowdb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: project })) .mockReturnValueOnce(createChain({ findIndex: 0 })); - (ContentTypesMapperModelLowdb.chain.get as ReturnType) + (mockContentTypesDb.chain.get as ReturnType) .mockReturnValueOnce(createChain({ find: contentType })) .mockReturnValueOnce(createChain({ findIndex: 0 })); - (FieldMapperModel.chain.get as ReturnType).mockReturnValue( + (mockFieldDb.chain.get as ReturnType).mockReturnValue( createChain({ findIndex: 0 }) ); diff --git a/api/tests/unit/services/migration.service.test.ts b/api/tests/unit/services/migration.service.test.ts index e3b25473f..3cb392550 100644 --- a/api/tests/unit/services/migration.service.test.ts +++ b/api/tests/unit/services/migration.service.test.ts @@ -1,4 +1,6 @@ +import path from 'path'; import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { MIGRATION_DATA_CONFIG } from '../../../src/constants/index.js'; const { mockHttps, @@ -12,6 +14,9 @@ const { mockFsPromisesReadFile, mockFsPromisesAppendFile, mockFsPromisesRealpath, + mockFsPromisesLstat, + mockFsMkdirSync, + mockFsWriteFileSync, } = vi.hoisted(() => { const projects = [ { @@ -37,6 +42,9 @@ const { mockFsPromisesReadFile: vi.fn(), mockFsPromisesAppendFile: vi.fn(), mockFsPromisesRealpath: vi.fn(), + mockFsPromisesLstat: vi.fn(), + mockFsMkdirSync: vi.fn(), + mockFsWriteFileSync: vi.fn(), }; }); @@ -146,8 +154,11 @@ vi.mock('fs', () => ({ default: { existsSync: (...args: unknown[]) => mockFsExistsSync(...args), readdirSync: (...args: unknown[]) => mockFsReadDirSync(...args), + mkdirSync: (...args: unknown[]) => mockFsMkdirSync(...args), + writeFileSync: (...args: unknown[]) => mockFsWriteFileSync(...args), promises: { readFile: (...args: unknown[]) => mockFsPromisesReadFile(...args), + lstat: (...args: unknown[]) => mockFsPromisesLstat(...args), }, }, })); @@ -157,6 +168,7 @@ vi.mock('fs/promises', () => ({ readFile: mockFsPromisesReadFile, appendFile: mockFsPromisesAppendFile, realpath: mockFsPromisesRealpath, + lstat: mockFsPromisesLstat, }, })); @@ -539,6 +551,33 @@ describe('migration.service', () => { findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(0) }), }); + const migrationDataBase = path.resolve(process.cwd(), MIGRATION_DATA_CONFIG.DATA); + const assetsIndexPath = path.join( + migrationDataBase, + 'dest-stack-1', + MIGRATION_DATA_CONFIG.ASSETS_DIR_NAME, + MIGRATION_DATA_CONFIG.ASSETS_SCHEMA_FILE, + ); + + mockFsPromisesLstat.mockResolvedValueOnce({ + isSymbolicLink: () => false, + isFile: () => true, + }); + mockFsPromisesRealpath.mockImplementation(async (p: string | URL) => { + const s = path.normalize(String(p)); + if (s === path.normalize(assetsIndexPath)) { + return assetsIndexPath; + } + throw new Error('File not found'); + }); + mockFsPromisesReadFile.mockImplementation(async (p: string | URL) => { + const s = path.normalize(String(p)); + if (s === path.normalize(assetsIndexPath)) { + return '{}'; + } + return ''; + }); + const req = createMockReq({ params: { orgId: 'org-123', projectId: 'proj-1' }, body: { token_payload: { region: 'NA', user_id: 'user-123', is_sso: false } }, @@ -546,6 +585,8 @@ describe('migration.service', () => { await expect(migrationService.startMigration(req)).resolves.not.toThrow(); expect(mockProjectUpdate).toHaveBeenCalled(); + expect(mockFsPromisesLstat).toHaveBeenCalled(); + expect(mockFsWriteFileSync).toHaveBeenCalled(); }); it('should do nothing when project has no destination_stack_id', async () => { diff --git a/api/tests/unit/services/projects.service.test.ts b/api/tests/unit/services/projects.service.test.ts index 364da1e68..12c980226 100644 --- a/api/tests/unit/services/projects.service.test.ts +++ b/api/tests/unit/services/projects.service.test.ts @@ -11,17 +11,51 @@ const { mockHttps, mockGetAuthToken, mockFindIndexValue, -} = vi.hoisted(() => ({ - mockProjectRead: vi.fn(), - mockProjectUpdate: vi.fn(), - mockProjectWrite: vi.fn(), - mockFindValue: vi.fn(), - mockFilterValue: vi.fn(), - mockGetProjectUtil: vi.fn(), - mockHttps: vi.fn(), - mockGetAuthToken: vi.fn(), - mockFindIndexValue: vi.fn(), -})); + mockContentTypesDb, + mockFieldDb, + getContentTypesMapperDbMock, + getFieldMapperDbMock, +} = vi.hoisted(() => { + const mockCtChainGet = vi.fn().mockReturnValue({ + filter: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue([]) }), + find: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(null) }), + findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(-1) }), + }); + const mockFieldChainGet = vi.fn().mockReturnValue({ + filter: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue([]) }), + find: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(null) }), + findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(-1) }), + }); + const mockContentTypesDb = { + read: vi.fn().mockResolvedValue(undefined), + update: vi.fn(), + write: vi.fn(), + chain: { get: mockCtChainGet }, + data: { ContentTypesMappers: [] as unknown[] }, + }; + const mockFieldDb = { + read: vi.fn().mockResolvedValue(undefined), + update: vi.fn(), + write: vi.fn(), + chain: { get: mockFieldChainGet }, + data: { field_mapper: [] as unknown[] }, + }; + return { + mockProjectRead: vi.fn(), + mockProjectUpdate: vi.fn(), + mockProjectWrite: vi.fn(), + mockFindValue: vi.fn(), + mockFilterValue: vi.fn(), + mockGetProjectUtil: vi.fn(), + mockHttps: vi.fn(), + mockGetAuthToken: vi.fn(), + mockFindIndexValue: vi.fn(), + mockContentTypesDb, + mockFieldDb, + getContentTypesMapperDbMock: vi.fn(() => mockContentTypesDb), + getFieldMapperDbMock: vi.fn(() => mockFieldDb), + }; +}); vi.mock('../../../src/models/project-lowdb.js', () => ({ default: { @@ -54,34 +88,11 @@ vi.mock('../../../src/config/index.js', () => ({ }, })); vi.mock('../../../src/models/contentTypesMapper-lowdb.js', () => ({ - default: { - read: vi.fn().mockResolvedValue(undefined), - update: vi.fn(), - write: vi.fn(), - chain: { - get: vi.fn().mockReturnValue({ - filter: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue([]) }), - find: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(null) }), - findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(-1) }), - }), - }, - data: { ContentTypesMappers: [] }, - }, + default: getContentTypesMapperDbMock, + getContentTypesMapperDb: getContentTypesMapperDbMock, })); vi.mock('../../../src/models/FieldMapper.js', () => ({ - default: { - read: vi.fn().mockResolvedValue(undefined), - update: vi.fn(), - write: vi.fn(), - chain: { - get: vi.fn().mockReturnValue({ - filter: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue([]) }), - find: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(null) }), - findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(-1) }), - }), - }, - data: { field_mapper: [] }, - }, + default: getFieldMapperDbMock, })); vi.mock('../../../src/services/contentMapper.service.js', () => ({ contentMapperService: { @@ -577,8 +588,7 @@ describe('projects.service', () => { const mockModel = await import('../../../src/models/project-lowdb.js'); (mockModel.default as any).data = { projects: [project] }; - const ctMock = await import('../../../src/models/contentTypesMapper-lowdb.js'); - (ctMock.default as any).chain.get.mockReturnValue({ + (mockContentTypesDb.chain.get as ReturnType).mockReturnValue({ find: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue({ id: 'ct-1', fieldMapping: [] }) }), findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(0) }), }); diff --git a/api/tests/unit/utils/asset-update.utils.test.ts b/api/tests/unit/utils/asset-update.utils.test.ts new file mode 100644 index 000000000..56de03d2d --- /dev/null +++ b/api/tests/unit/utils/asset-update.utils.test.ts @@ -0,0 +1,396 @@ +import path from 'node:path'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { + mockProjectRead, + mockChainGet, + mockExistsSync, + mockReadFileSync, + mockWriteFileSync, + mockMkdirSync, + mockReaddirSync, + mockRmSync, + mockAppendFileSync, +} = vi.hoisted(() => ({ + mockProjectRead: vi.fn(), + mockChainGet: vi.fn(), + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockWriteFileSync: vi.fn(), + mockMkdirSync: vi.fn(), + mockReaddirSync: vi.fn(), + mockRmSync: vi.fn(), + mockAppendFileSync: vi.fn(), +})); + +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { + read: mockProjectRead, + chain: { get: mockChainGet }, + }, +})); + +vi.mock('node:fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + writeFileSync: mockWriteFileSync, + mkdirSync: mockMkdirSync, + readdirSync: mockReaddirSync, + rmSync: mockRmSync, + appendFileSync: mockAppendFileSync, + }, +})); + +const project = (opts: Partial<{ iteration: number; destination_stack_id: string | undefined }>) => { + const p = { + id: 'p1', + iteration: opts.iteration ?? 1, + destination_stack_id: 'stack1' as string | undefined, + }; + if ('destination_stack_id' in opts) { + p.destination_stack_id = opts.destination_stack_id; + } + return p; +}; + +describe('asset-update.utils', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockProjectRead.mockResolvedValue(undefined); + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({})), + }), + }); + }); + + it('removeExistingAssets returns early when no stackId', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ destination_stack_id: '' })), + }), + }); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + expect(mockExistsSync).not.toHaveBeenCalled(); + }); + + it('removeExistingAssets returns when assets index.json missing', async () => { + mockExistsSync.mockReturnValue(false); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + expect(mockReadFileSync).not.toHaveBeenCalled(); + }); + + it('removeExistingAssets returns when index file empty', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(' '); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets iteration 1 saves metadata only', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue(JSON.stringify({ a1: { filename: 'f', file_size: '1', url: 'u' } })); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1', '/tmp/a.log'); + expect(mockMkdirSync).toHaveBeenCalled(); + expect(mockWriteFileSync).toHaveBeenCalled(); + }); + + it('removeExistingAssets iteration 2+ skips dedup when no previous uid map', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + mockExistsSync.mockImplementation((p: string) => { + if (String(p).endsWith('index.json')) return true; + return false; + }); + mockReadFileSync.mockReturnValue(JSON.stringify({ a1: { filename: 'f', file_size: '1' } })); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets iteration 2+ runs dedup when prev maps exist', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + let indexRead = false; + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes(`${path.sep}entries${path.sep}`)) return true; + if (s.includes('files')) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json') && !indexRead) { + indexRead = true; + return JSON.stringify({ + asset1: { filename: 'f', file_size: '1' }, + }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { asset1: 'cs-uid-1' } }); + } + if (s.includes('asset-metadata.json')) { + return JSON.stringify({ + asset1: { filename: 'f', file_size: '1', url: '' }, + }); + } + if (s.endsWith('.json') && s.includes('entries')) { + return JSON.stringify({ ref: { uid: 'asset1' } }); + } + return '{}'; + }); + const dirent = (name: string, dir: boolean) => ({ name, isDirectory: () => dir }); + mockReaddirSync + .mockReturnValueOnce([dirent('ct', true)]) + .mockReturnValueOnce([dirent('en', true)]) + .mockReturnValueOnce(['e.json']); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + expect(mockWriteFileSync).toHaveBeenCalled(); + }); + + it('removeExistingAssets returns when index JSON parse fails', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue('{ not json'); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets iteration 2+ exits when no unchanged assets to dedupe', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + let call = 0; + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json') && call === 0) { + call += 1; + return JSON.stringify({ a1: { filename: 'new-name', file_size: '1' } }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { a1: 'cs-1' } }); + } + if (s.includes('asset-metadata.json')) { + return JSON.stringify({ + a1: { filename: 'old-name', file_size: '1', url: '' }, + }); + } + return '{}'; + }); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets tolerates corrupt previous asset-metadata JSON', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) { + return JSON.stringify({ x1: { filename: 'f', file_size: '1' } }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { x1: 'cs-1' } }); + } + if (s.includes('asset-metadata.json')) { + return '{ bad json'; + } + return '{}'; + }); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets tolerates corrupt uid-mapper JSON', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) { + return JSON.stringify({ a1: { filename: 'f', file_size: '1' } }); + } + if (s.includes('uid-mapper.json')) { + return 'not-json'; + } + return '{}'; + }); + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets removes folders and updates index when dedup applies', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + let indexPass = 0; + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + if (s.includes(`${path.sep}entries${path.sep}`)) return true; + if (s.endsWith(`${path.sep}files`) || s.includes(`${path.sep}files${path.sep}`)) return true; + if (s.includes(`${path.sep}files${path.sep}a1`)) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) { + indexPass += 1; + return JSON.stringify({ a1: { filename: 'f', file_size: '1' } }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { a1: 'cs-uid-1' } }); + } + if (s.includes('asset-metadata.json')) { + return JSON.stringify({ + a1: { filename: 'f', file_size: '1', url: '' }, + }); + } + if (s.endsWith('.json') && s.includes('entries')) { + return JSON.stringify({ nested: { uid: 'a1' } }); + } + return '{}'; + }); + const dirent = (name: string, dir: boolean) => ({ name, isDirectory: () => dir }); + mockReaddirSync + .mockReturnValueOnce([dirent('ct', true)]) + .mockReturnValueOnce([dirent('en', true)]) + .mockReturnValueOnce(['e.json']); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + expect(mockRmSync).toHaveBeenCalled(); + }); + + it('removeExistingAssets skips entry JSON files that fail to parse', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + if (s.includes(`${path.sep}entries${path.sep}`)) return true; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) { + return JSON.stringify({ z1: { filename: 'f', file_size: '1' } }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { z1: 'cs-z' } }); + } + if (s.includes('asset-metadata.json')) { + return JSON.stringify({ z1: { filename: 'f', file_size: '1', url: '' } }); + } + if (s.endsWith('bad.json')) { + return '{'; + } + return '{}'; + }); + const dirent = (name: string, dir: boolean) => ({ name, isDirectory: () => dir }); + mockReaddirSync + .mockReturnValueOnce([dirent('ct', true)]) + .mockReturnValueOnce([dirent('en', true)]) + .mockReturnValueOnce(['bad.json']); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + }); + + it('removeExistingAssets replaces nested asset uids and skips empty entry JSON', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue(project({ iteration: 2 })), + }), + }); + let entryReadCount = 0; + mockExistsSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) return true; + if (s.includes('uid-mapper.json')) return true; + if (s.includes('asset-metadata.json')) return true; + if (s.includes(`${path.sep}entries${path.sep}`)) return true; + if (s.includes(`${path.sep}files`)) return false; + return false; + }); + mockReadFileSync.mockImplementation((p: string) => { + const s = String(p); + if (s.endsWith('index.json')) { + return JSON.stringify({ a1: { filename: 'f', file_size: '1' } }); + } + if (s.includes('uid-mapper.json')) { + return JSON.stringify({ assets: { a1: 'cs-nested' } }); + } + if (s.includes('asset-metadata.json')) { + return JSON.stringify({ + a1: { filename: 'f', file_size: '1', url: '' }, + }); + } + if (s.endsWith('empty.json')) { + return ' \n'; + } + if (s.endsWith('nested.json')) { + return JSON.stringify({ level: { deeper: { uid: 'a1' } } }); + } + return '{}'; + }); + const dirent = (name: string, dir: boolean) => ({ name, isDirectory: () => dir }); + mockReaddirSync + .mockReturnValueOnce([dirent('ct', true)]) + .mockReturnValueOnce([dirent('en', true)]) + .mockReturnValueOnce(['empty.json', 'nested.json']); + + const { removeExistingAssets } = await import('../../../src/utils/asset-update.utils.js'); + await removeExistingAssets('p1'); + expect(mockWriteFileSync).toHaveBeenCalled(); + }); +}); diff --git a/api/tests/unit/utils/auth.utils.test.ts b/api/tests/unit/utils/auth.utils.test.ts index 26a006ed9..659b24103 100644 --- a/api/tests/unit/utils/auth.utils.test.ts +++ b/api/tests/unit/utils/auth.utils.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const { mockRead, mockChain } = vi.hoisted(() => { +const { mockRead, mockChain, mockExistsSync, mockReadFileSync, mockDecryptAppConfig } = vi.hoisted(() => { const mockChain = { get: vi.fn().mockReturnThis(), findIndex: vi.fn().mockReturnThis(), @@ -9,6 +9,24 @@ const { mockRead, mockChain } = vi.hoisted(() => { return { mockRead: vi.fn(), mockChain, + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockDecryptAppConfig: vi.fn((c: Record) => c), + }; +}); + +vi.mock('fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + }, +})); + +vi.mock('../../../src/utils/crypto.utils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + decryptAppConfig: (c: Record) => mockDecryptAppConfig(c), }; }); @@ -26,12 +44,21 @@ vi.mock('../../../src/utils/custom-errors.utils.js', async (importOriginal) => { }); import getAuthToken from '../../../src/utils/auth.utils.js'; +import { + getAccessToken, + getAppOrganizationUID, + getAppOrganization, + getAppConfig, +} from '../../../src/utils/auth.utils.js'; import AuthenticationModel from '../../../src/models/authentication.js'; describe('auth.utils', () => { beforeEach(() => { vi.clearAllMocks(); mockRead.mockResolvedValue(undefined); + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockReturnValue('{}'); + mockDecryptAppConfig.mockImplementation((c: Record) => c); }); it('should return authtoken for a valid user', async () => { @@ -60,4 +87,62 @@ describe('auth.utils', () => { await expect(getAuthToken('NA', 'user-123')).rejects.toThrow(); }); + + describe('getAccessToken', () => { + it('returns access_token for valid user', async () => { + mockChain.value.mockReturnValue(0); + (AuthenticationModel as any).data = { + users: [{ region: 'EU', user_id: 'u2', access_token: 'at-1' }], + }; + await expect(getAccessToken('EU', 'u2')).resolves.toBe('at-1'); + }); + + it('throws when user missing or no access_token', async () => { + mockChain.value.mockReturnValue(-1); + (AuthenticationModel as any).data = { users: [] }; + await expect(getAccessToken('NA', 'x')).rejects.toThrow(); + }); + }); + + describe('loadAppConfig consumers', () => { + it('getAppOrganizationUID returns uid', () => { + mockReadFileSync.mockReturnValue( + JSON.stringify({ organization: { uid: 'org-uid', name: 'N' } }) + ); + expect(getAppOrganizationUID()).toBe('org-uid'); + }); + + it('getAppOrganizationUID throws when uid missing', () => { + mockReadFileSync.mockReturnValue(JSON.stringify({ organization: { name: 'N' } })); + expect(() => getAppOrganizationUID()).toThrow('Organization UID not found'); + }); + + it('getAppOrganization returns uid and name', () => { + mockReadFileSync.mockReturnValue( + JSON.stringify({ organization: { uid: 'o1', name: 'Org' } }) + ); + expect(getAppOrganization()).toEqual({ uid: 'o1', name: 'Org' }); + }); + + it('getAppOrganization throws when org incomplete', () => { + mockReadFileSync.mockReturnValue(JSON.stringify({ organization: { uid: 'o1' } })); + expect(() => getAppOrganization()).toThrow('Organization details not found'); + }); + + it('getAppConfig returns config when oauthData present', () => { + const cfg = { oauthData: { client_id: 'c' } }; + mockReadFileSync.mockReturnValue(JSON.stringify(cfg)); + expect(getAppConfig()).toMatchObject(cfg); + }); + + it('getAppConfig throws when oauthData missing', () => { + mockReadFileSync.mockReturnValue(JSON.stringify({})); + expect(() => getAppConfig()).toThrow('SSO is not configured'); + }); + + it('loadAppConfig throws when app.json missing', () => { + mockExistsSync.mockReturnValue(false); + expect(() => getAppOrganizationUID()).toThrow('app.json file not found'); + }); + }); }); diff --git a/api/tests/unit/utils/batch-processor.utils.test.ts b/api/tests/unit/utils/batch-processor.utils.test.ts index 65855d596..f2d28da8a 100644 --- a/api/tests/unit/utils/batch-processor.utils.test.ts +++ b/api/tests/unit/utils/batch-processor.utils.test.ts @@ -1,7 +1,11 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, afterEach } from 'vitest'; import { BatchProcessor, processBatches } from '../../../src/utils/batch-processor.utils.js'; describe('batch-processor.utils', () => { + afterEach(() => { + delete (globalThis as unknown as { gc?: () => void }).gc; + }); + describe('BatchProcessor', () => { it('should process all items in correct batch sizes', async () => { const processor = new BatchProcessor({ @@ -84,6 +88,18 @@ describe('batch-processor.utils', () => { expect(elapsed).toBeGreaterThanOrEqual(80); }); + + it('calls global.gc when available after each batch', async () => { + const gc = vi.fn(); + (globalThis as unknown as { gc: () => void }).gc = gc; + const processor = new BatchProcessor({ + batchSize: 1, + concurrency: 1, + delayBetweenBatches: 0, + }); + await processor.processBatches([1, 2], async (item) => item); + expect(gc).toHaveBeenCalled(); + }); }); describe('processBatches utility function', () => { diff --git a/api/tests/unit/utils/config-handler.util.test.ts b/api/tests/unit/utils/config-handler.util.test.ts new file mode 100644 index 000000000..abf5562cc --- /dev/null +++ b/api/tests/unit/utils/config-handler.util.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockSet = vi.fn(); + +vi.mock('@contentstack/cli-utilities', () => ({ + configHandler: { + set: mockSet, + }, +})); + +describe('config-handler.util', () => { + beforeEach(() => { + mockSet.mockClear(); + }); + + it('setOAuthConfig writes expected keys', async () => { + const { setOAuthConfig } = await import('../../../src/utils/config-handler.util.js'); + const userData = { + access_token: 'at', + refresh_token: 'rt', + updated_at: '2024-01-01', + email: 'a@b.com', + user_id: 'u1', + organization_uid: 'org1', + }; + setOAuthConfig(userData); + expect(mockSet).toHaveBeenCalledWith('oauthAccessToken', 'at'); + expect(mockSet).toHaveBeenCalledWith('oauthRefreshToken', 'rt'); + expect(mockSet).toHaveBeenCalledWith('authorisationType', 'OAUTH'); + }); + + it('setOAuthConfig uses created_at when updated_at missing', async () => { + const { setOAuthConfig } = await import('../../../src/utils/config-handler.util.js'); + setOAuthConfig({ + access_token: 'a', + refresh_token: 'b', + created_at: '2023-01-01', + email: 'e', + user_id: 'u', + organization_uid: 'o', + }); + expect(mockSet).toHaveBeenCalledWith('oauthDateTime', '2023-01-01'); + }); + + it('setBasicAuthConfig writes authtoken and BASIC', async () => { + const { setBasicAuthConfig } = await import('../../../src/utils/config-handler.util.js'); + setBasicAuthConfig({ authtoken: 'tok', email: 'e@e.com' }); + expect(mockSet).toHaveBeenCalledWith('authtoken', 'tok'); + expect(mockSet).toHaveBeenCalledWith('authorisationType', 'BASIC'); + }); + + it('setOAuthConfig uses Date when neither updated_at nor created_at is set', async () => { + const { setOAuthConfig } = await import('../../../src/utils/config-handler.util.js'); + setOAuthConfig({ + access_token: 'a', + refresh_token: 'b', + email: 'e', + user_id: 'u', + organization_uid: 'o', + }); + expect(mockSet).toHaveBeenCalledWith('oauthDateTime', expect.any(Date)); + }); + + it('setOAuthConfig tolerates empty object', async () => { + const { setOAuthConfig } = await import('../../../src/utils/config-handler.util.js'); + setOAuthConfig({}); + expect(mockSet).toHaveBeenCalledWith('oauthAccessToken', undefined); + }); + + it('setBasicAuthConfig tolerates empty object', async () => { + const { setBasicAuthConfig } = await import('../../../src/utils/config-handler.util.js'); + setBasicAuthConfig({}); + expect(mockSet).toHaveBeenCalledWith('authtoken', undefined); + expect(mockSet).toHaveBeenCalledWith('email', undefined); + }); +}); diff --git a/api/tests/unit/utils/content-type-checker.utils.test.ts b/api/tests/unit/utils/content-type-checker.utils.test.ts new file mode 100644 index 000000000..df4573a04 --- /dev/null +++ b/api/tests/unit/utils/content-type-checker.utils.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockRead = vi.fn(); +const mockExistsSync = vi.fn(); +const mockContentTypesData = vi.hoisted(() => ({ + value: { ContentTypesMappers: [{ otherCmsUid: 'ct-uid' }] } as any, +})); + +vi.mock('fs', () => ({ + default: { existsSync: mockExistsSync }, + existsSync: mockExistsSync, +})); + +vi.mock('../../../src/models/contentTypesMapper-lowdb.js', () => ({ + default: vi.fn(() => ({ + read: mockRead, + data: mockContentTypesData.value, + })), +})); + +describe('content-type-checker.utils', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockRead.mockResolvedValue(undefined); + mockExistsSync.mockReturnValue(true); + mockContentTypesData.value = { ContentTypesMappers: [{ otherCmsUid: 'ct-uid' }] }; + }); + + it('isContentTypeAlreadyCreated returns false when currentIteration <= 1', async () => { + const { isContentTypeAlreadyCreated } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(isContentTypeAlreadyCreated('p1', 'x', 1)).resolves.toBe(false); + expect(mockExistsSync).not.toHaveBeenCalled(); + }); + + it('isContentTypeAlreadyCreated returns true when prior iteration has matching uid', async () => { + const { isContentTypeAlreadyCreated } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(isContentTypeAlreadyCreated('p1', 'ct-uid', 3)).resolves.toBe(true); + expect(mockRead).toHaveBeenCalled(); + }); + + it('getPreviouslyCreatedContentTypes returns empty when iteration <= 1', async () => { + const { getPreviouslyCreatedContentTypes } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(getPreviouslyCreatedContentTypes('p1', 1)).resolves.toEqual([]); + }); + + it('shouldSkipContentTypeCreation delegates to isContentTypeAlreadyCreated', async () => { + const mod = await import('../../../src/utils/content-type-checker.utils.js'); + await expect(mod.shouldSkipContentTypeCreation('p1', 'ct-uid', 1)).resolves.toBe(false); + }); + + it('isContentTypeAlreadyCreated skips iterations with no directory', async () => { + mockExistsSync.mockReturnValue(false); + const { isContentTypeAlreadyCreated } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(isContentTypeAlreadyCreated('p1', 'ct-uid', 3)).resolves.toBe(false); + }); + + it('isContentTypeAlreadyCreated continues when read throws', async () => { + mockExistsSync.mockReturnValue(true); + mockRead.mockRejectedValueOnce(new Error('read fail')); + const { isContentTypeAlreadyCreated } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(isContentTypeAlreadyCreated('p1', 'missing', 3)).resolves.toBe(false); + }); + + it('getPreviouslyCreatedContentTypes collects uids from prior iterations', async () => { + const { getPreviouslyCreatedContentTypes } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(getPreviouslyCreatedContentTypes('p1', 3)).resolves.toContain('ct-uid'); + }); + + it('getPreviouslyCreatedContentTypes continues when iteration read fails', async () => { + mockExistsSync.mockReturnValue(true); + mockRead.mockRejectedValue(new Error('boom')); + const { getPreviouslyCreatedContentTypes } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(getPreviouslyCreatedContentTypes('p1', 2)).resolves.toEqual([]); + }); + + it('getPreviouslyCreatedContentTypes skips iterations with missing directories', async () => { + mockExistsSync.mockReturnValue(false); + const { getPreviouslyCreatedContentTypes } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(getPreviouslyCreatedContentTypes('p1', 3)).resolves.toEqual([]); + expect(mockRead).not.toHaveBeenCalled(); + }); + + it('getPreviouslyCreatedContentTypes returns empty when mapper data is missing', async () => { + mockContentTypesData.value = undefined; + const { getPreviouslyCreatedContentTypes } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(getPreviouslyCreatedContentTypes('p1', 2)).resolves.toEqual([]); + }); + + it('getPreviouslyCreatedContentTypes ignores entries without otherCmsUid', async () => { + mockContentTypesData.value = { ContentTypesMappers: [{}, { otherCmsUid: 'ct-uid-2' }] }; + const { getPreviouslyCreatedContentTypes } = await import( + '../../../src/utils/content-type-checker.utils.js' + ); + await expect(getPreviouslyCreatedContentTypes('p1', 2)).resolves.toEqual(['ct-uid-2']); + }); +}); diff --git a/api/tests/unit/utils/crypto.utils.test.ts b/api/tests/unit/utils/crypto.utils.test.ts new file mode 100644 index 000000000..da4ec8f86 --- /dev/null +++ b/api/tests/unit/utils/crypto.utils.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +describe('crypto.utils', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + vi.resetModules(); + process.env.MANIFEST_ENCRYPT_KEY = 'test-key-32-chars-long-string!!'; + process.env.MANIFEST_ENCRYPT_SALT = 'testsalt'; + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('decrypt returns plain values unchanged when not prefixed', async () => { + const { decrypt } = await import('../../../src/utils/crypto.utils.js'); + expect(decrypt('plain')).toBe('plain'); + expect(decrypt('')).toBe(''); + expect(decrypt(' no-enc-prefix')).toBe(' no-enc-prefix'); + }); + + it('decryptAppConfig returns config unchanged when oauthData/pkce absent', async () => { + const { decryptAppConfig } = await import('../../../src/utils/crypto.utils.js'); + const cfg = { foo: 'bar' }; + expect(decryptAppConfig(cfg)).toBe(cfg); + }); + + it('getEncryptKey throws when MANIFEST_ENCRYPT_KEY missing', async () => { + vi.resetModules(); + delete process.env.MANIFEST_ENCRYPT_KEY; + process.env.MANIFEST_ENCRYPT_SALT = 'testsalt'; + const { decrypt } = await import('../../../src/utils/crypto.utils.js'); + expect(() => decrypt('enc:00112233445566778899aabb:00112233445566778899aabb:445566')).toThrow( + 'MANIFEST_ENCRYPT_KEY' + ); + }); + + it('decrypt throws on invalid enc: format (wrong segment count)', async () => { + const { decrypt } = await import('../../../src/utils/crypto.utils.js'); + expect(() => decrypt('enc:only:two')).toThrow('Invalid encrypted value format'); + }); + + it('getEncryptSalt throws when MANIFEST_ENCRYPT_SALT missing', async () => { + vi.resetModules(); + process.env.MANIFEST_ENCRYPT_KEY = 'test-key-32-chars-long-string!!'; + delete process.env.MANIFEST_ENCRYPT_SALT; + const { decrypt } = await import('../../../src/utils/crypto.utils.js'); + expect(() => decrypt('enc:aa:bb:ccdd')).toThrow('MANIFEST_ENCRYPT_SALT'); + }); + + it('decryptAppConfig runs oauthData and pkce decrypt branches for plain strings', async () => { + const { decryptAppConfig } = await import('../../../src/utils/crypto.utils.js'); + const cfg = { + oauthData: { + client_id: 'id-plain', + client_secret: 'sec-plain', + }, + pkce: { + code_verifier: 'ver-plain', + code_challenge: 'chal-plain', + }, + }; + const out = decryptAppConfig({ ...cfg }); + expect(out.oauthData?.client_id).toBe('id-plain'); + expect(out.pkce?.code_verifier).toBe('ver-plain'); + }); + + it('decryptAppConfig decrypts client_id only when client_secret absent', async () => { + const { decryptAppConfig } = await import('../../../src/utils/crypto.utils.js'); + const out = decryptAppConfig({ + oauthData: { client_id: 'only-id' }, + } as Record); + expect((out as any).oauthData.client_secret).toBeUndefined(); + expect((out as any).oauthData.client_id).toBe('only-id'); + }); + + it('decryptAppConfig handles pkce with only code_challenge', async () => { + const { decryptAppConfig } = await import('../../../src/utils/crypto.utils.js'); + const out = decryptAppConfig({ + pkce: { code_challenge: 'chal-only' }, + } as Record); + expect((out as any).pkce.code_challenge).toBe('chal-only'); + }); + + it('decryptAppConfig handles pkce with only code_verifier', async () => { + const { decryptAppConfig } = await import('../../../src/utils/crypto.utils.js'); + const out = decryptAppConfig({ + pkce: { code_verifier: 'ver-only' }, + } as Record); + expect((out as any).pkce.code_verifier).toBe('ver-only'); + }); + + it('decryptAppConfig skips inner oauth fields when oauthData is empty object', async () => { + const { decryptAppConfig } = await import('../../../src/utils/crypto.utils.js'); + const out = decryptAppConfig({ oauthData: {} } as Record); + expect(out.oauthData).toEqual({}); + }); +}); diff --git a/api/tests/unit/utils/custom-errors.utils.test.ts b/api/tests/unit/utils/custom-errors.utils.test.ts index 2b4c45a36..cc1d9cff2 100644 --- a/api/tests/unit/utils/custom-errors.utils.test.ts +++ b/api/tests/unit/utils/custom-errors.utils.test.ts @@ -58,6 +58,11 @@ describe('Custom Error Classes', () => { expect(error.message).toBe('DB error'); expect(error).toBeInstanceOf(AppError); }); + + it('should accept a custom message', () => { + const error = new DatabaseError('custom-db'); + expect(error.message).toBe('custom-db'); + }); }); describe('ValidationError', () => { @@ -67,6 +72,11 @@ describe('Custom Error Classes', () => { expect(error.message).toBe('User validation error'); expect(error).toBeInstanceOf(AppError); }); + + it('should accept a custom message', () => { + const error = new ValidationError('bad field'); + expect(error.message).toBe('bad field'); + }); }); describe('InternalServerError', () => { @@ -76,6 +86,11 @@ describe('Custom Error Classes', () => { expect(error.message).toBeTruthy(); expect(error).toBeInstanceOf(AppError); }); + + it('should accept a custom message', () => { + const error = new InternalServerError('custom-internal'); + expect(error.message).toBe('custom-internal'); + }); }); describe('UnauthorizedError', () => { @@ -97,6 +112,11 @@ describe('Custom Error Classes', () => { expect(error.statusCode).toBe(500); expect(error).toBeInstanceOf(AppError); }); + + it('should accept a custom message', () => { + const error = new S3Error('bucket failed'); + expect(error.message).toBe('bucket failed'); + }); }); describe('ExceptionFunction', () => { diff --git a/api/tests/unit/utils/entry-duplicate.utils.test.ts b/api/tests/unit/utils/entry-duplicate.utils.test.ts new file mode 100644 index 000000000..428db859e --- /dev/null +++ b/api/tests/unit/utils/entry-duplicate.utils.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockProjectRead = vi.fn(); +const mockChainGet = vi.fn(); +const mockEntryRead = vi.fn(); +const mockEntryUpdate = vi.fn(); + +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { + read: mockProjectRead, + chain: { get: mockChainGet }, + }, +})); + +vi.mock('../../../src/models/EntryMapper.js', () => ({ + default: vi.fn(() => ({ + read: mockEntryRead, + update: mockEntryUpdate, + })), +})); + +describe('entry-duplicate.utils', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockProjectRead.mockResolvedValue(undefined); + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue({ id: 'p1', iteration: 2 }), + }), + }); + mockEntryRead.mockResolvedValue(undefined); + }); + + it('marks duplicate entries sharing contentTypeId, language, and entryName', async () => { + const rows = [ + { contentTypeId: 'ct', language: 'en', entryName: 'e1', isDuplicateEntry: false }, + { contentTypeId: 'ct', language: 'en', entryName: 'e1', isDuplicateEntry: false }, + ]; + mockEntryUpdate.mockImplementation(async (fn: (d: { entry_mapper: typeof rows }) => void) => { + fn({ entry_mapper: rows }); + }); + + const { isDuplicateEntry } = await import('../../../src/utils/entry-duplicate.utils.js'); + await isDuplicateEntry('p1'); + + expect(rows[0].isDuplicateEntry).toBe(true); + expect(rows[1].isDuplicateEntry).toBe(true); + }); + + it('leaves unique entries unchanged', async () => { + const rows = [ + { contentTypeId: 'ct', language: 'en', entryName: 'a', isDuplicateEntry: false }, + { contentTypeId: 'ct', language: 'en', entryName: 'b', isDuplicateEntry: false }, + ]; + mockEntryUpdate.mockImplementation(async (fn: (d: { entry_mapper: typeof rows }) => void) => { + fn({ entry_mapper: rows }); + }); + + const { isDuplicateEntry } = await import('../../../src/utils/entry-duplicate.utils.js'); + await isDuplicateEntry('p1'); + + expect(rows.every((r) => !r.isDuplicateEntry)).toBe(true); + }); +}); diff --git a/api/tests/unit/utils/entry-update.utils.test.ts b/api/tests/unit/utils/entry-update.utils.test.ts new file mode 100644 index 000000000..dc9e39ed9 --- /dev/null +++ b/api/tests/unit/utils/entry-update.utils.test.ts @@ -0,0 +1,147 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { + mockProjectRead, + mockChainGet, + mockEntryRead, + mockEntryChainGet, + mockExistsSync, + mockReadFileSync, + mockWriteFileSync, + mockMkdirSync, + mockReaddirSync, + mockAppendFileSync, +} = vi.hoisted(() => ({ + mockProjectRead: vi.fn(), + mockChainGet: vi.fn(), + mockEntryRead: vi.fn(), + mockEntryChainGet: vi.fn(), + mockExistsSync: vi.fn(), + mockReadFileSync: vi.fn(), + mockWriteFileSync: vi.fn(), + mockMkdirSync: vi.fn(), + mockReaddirSync: vi.fn(), + mockAppendFileSync: vi.fn(), +})); + +vi.mock('../../../src/models/project-lowdb.js', () => ({ + default: { + read: mockProjectRead, + chain: { get: mockChainGet }, + }, +})); + +vi.mock('../../../src/models/EntryMapper.js', () => ({ + default: vi.fn(() => ({ + read: mockEntryRead, + chain: { get: mockEntryChainGet }, + })), +})); + +vi.mock('node:fs', () => ({ + default: { + existsSync: mockExistsSync, + readFileSync: mockReadFileSync, + writeFileSync: mockWriteFileSync, + mkdirSync: mockMkdirSync, + appendFileSync: mockAppendFileSync, + readdirSync: mockReaddirSync, + }, +})); + +describe('entry-update.utils', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockProjectRead.mockResolvedValue(undefined); + mockEntryRead.mockResolvedValue(undefined); + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue({ + id: 'p1', + iteration: 1, + destination_stack_id: 'stack1', + }), + }), + }); + mockEntryChainGet.mockReturnValue({ + value: () => [ + { otherCmsEntryUid: 'legacy-key', isUpdate: true, contentstackEntryUid: 'cs-uid' }, + ], + }); + }); + + it('removeEntriesFromDatabase returns null when stackId missing', async () => { + mockChainGet.mockReturnValue({ + find: vi.fn().mockReturnValue({ + value: vi.fn().mockReturnValue({ id: 'p1', iteration: 1 }), + }), + }); + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + await expect(removeEntriesFromDatabase('p1')).resolves.toBeNull(); + }); + + it('removeEntriesFromDatabase returns null when no entry_mapper items', async () => { + mockEntryChainGet.mockReturnValue({ value: () => [] }); + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + await expect(removeEntriesFromDatabase('p1')).resolves.toBeNull(); + }); + + it('removeEntriesFromDatabase returns null when entries directory missing', async () => { + mockExistsSync.mockReturnValue(false); + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + await expect(removeEntriesFromDatabase('p1')).resolves.toBeNull(); + }); + + it('removeEntriesFromDatabase walks dirs, updates json, writes config', async () => { + mockExistsSync.mockReturnValue(true); + const dirent = (name: string, isDir: boolean) => ({ + name, + isDirectory: () => isDir, + }); + mockReaddirSync + .mockReturnValueOnce([dirent('ct1', true)]) + .mockReturnValueOnce([dirent('en', true)]) + .mockReturnValueOnce(['page.json']); + mockReadFileSync.mockReturnValue( + JSON.stringify({ 'legacy-key': { title: 'Hello' }, keep: { x: 1 } }) + ); + + const { removeEntriesFromDatabase } = await import('../../../src/utils/entry-update.utils.js'); + const result = await removeEntriesFromDatabase('p1', '/tmp/mig.log'); + + expect(result).toMatch(/updated-entries\.json$/); + expect(mockWriteFileSync).toHaveBeenCalled(); + expect(mockMkdirSync).toHaveBeenCalled(); + }); + + it('enrichConfigWithAssetMapping covers iteration 1 (no old path branch)', async () => { + mockExistsSync.mockReturnValue(false); + const { enrichConfigWithAssetMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithAssetMapping('/c.json', 'proj', 1); + }); + + it('enrichConfigWithAssetMapping loads old and new uid-mappers when files exist', async () => { + mockExistsSync.mockImplementation((p: string) => + String(p).includes('uid-mapper.json') + ); + mockReadFileSync.mockReturnValue(JSON.stringify({ assets: { x: 'y' } })); + + const { enrichConfigWithAssetMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithAssetMapping('/c.json', 'proj', 2, '/log'); + }); + + it('enrichConfigWithAssetMapping catches JSON errors on old mapper', async () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => {}); + let calls = 0; + mockExistsSync.mockImplementation(() => true); + mockReadFileSync.mockImplementation(() => { + calls += 1; + if (calls === 1) throw new Error('bad json'); + return JSON.stringify({ assets: {} }); + }); + + const { enrichConfigWithAssetMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithAssetMapping('/c.json', 'proj', 2); + log.mockRestore(); + }); +}); diff --git a/api/tests/unit/utils/field-attacher.utils.test.ts b/api/tests/unit/utils/field-attacher.utils.test.ts index c4fcc32b9..a7f6d493e 100644 --- a/api/tests/unit/utils/field-attacher.utils.test.ts +++ b/api/tests/unit/utils/field-attacher.utils.test.ts @@ -8,6 +8,7 @@ const { mockContentTypesChain, mockFieldMapperChain, mockContenTypeMaker, + mockShouldSkipContentTypeCreation, } = vi.hoisted(() => { const mockProjectChain = { get: vi.fn().mockReturnThis(), @@ -32,6 +33,7 @@ const { mockContentTypesChain, mockFieldMapperChain, mockContenTypeMaker: vi.fn(), + mockShouldSkipContentTypeCreation: vi.fn(), }; }); @@ -43,23 +45,27 @@ vi.mock('../../../src/models/project-lowdb.js', () => ({ })); vi.mock('../../../src/models/contentTypesMapper-lowdb.js', () => ({ - default: { + default: () => ({ read: mockContentTypesRead, chain: mockContentTypesChain, - }, + }), })); vi.mock('../../../src/models/FieldMapper.js', () => ({ - default: { + default: () => ({ read: mockFieldMapperRead, chain: mockFieldMapperChain, - }, + }), })); vi.mock('../../../src/utils/content-type-creator.utils.js', () => ({ contenTypeMaker: (...args: any[]) => mockContenTypeMaker(...args), })); +vi.mock('../../../src/utils/content-type-checker.utils.js', () => ({ + shouldSkipContentTypeCreation: (...args: any[]) => mockShouldSkipContentTypeCreation(...args), +})); + import { fieldAttacher } from '../../../src/utils/field-attacher.utils.js'; describe('field-attacher.utils', () => { @@ -69,6 +75,7 @@ describe('field-attacher.utils', () => { mockContentTypesRead.mockResolvedValue(undefined); mockFieldMapperRead.mockResolvedValue(undefined); mockContenTypeMaker.mockResolvedValue(undefined); + mockShouldSkipContentTypeCreation.mockResolvedValue(false); }); it('should return empty array when project has no content_mapper', async () => { @@ -221,4 +228,58 @@ describe('field-attacher.utils', () => { expect(mockContenTypeMaker).toHaveBeenCalledTimes(2); expect(result).toHaveLength(2); }); + + it('should create content type in later iterations when skip check returns false', async () => { + const contentType = { id: 'ct-1', otherCmsUid: 'blog', fieldMapping: [] }; + + mockProjectChain.value.mockReturnValue({ + id: 'proj-1', + org_id: 'org-1', + iteration: 2, + content_mapper: ['ct-1'], + stackDetails: { isNewStack: false }, + mapperKeys: {}, + }); + mockContentTypesChain.value.mockReturnValue(contentType); + mockShouldSkipContentTypeCreation.mockResolvedValue(false); + + const result = await fieldAttacher({ + projectId: 'proj-1', + orgId: 'org-1', + destinationStackId: 'stack-1', + region: 'NA', + user_id: 'user-1', + is_sso: true, + }); + + expect(mockShouldSkipContentTypeCreation).toHaveBeenCalledWith('proj-1', 'blog', 2); + expect(mockContenTypeMaker).toHaveBeenCalledTimes(1); + expect(mockContenTypeMaker).toHaveBeenCalledWith(expect.objectContaining({ is_sso: true })); + expect(result).toHaveLength(1); + }); + + it('should skip content type creation in later iterations when skip check returns true', async () => { + mockProjectChain.value.mockReturnValue({ + id: 'proj-1', + org_id: 'org-1', + iteration: 2, + content_mapper: ['ct-1'], + stackDetails: { isNewStack: false }, + mapperKeys: {}, + }); + mockContentTypesChain.value.mockReturnValue({ id: 'ct-1', otherCmsUid: 'blog', fieldMapping: [] }); + mockShouldSkipContentTypeCreation.mockResolvedValue(true); + + const result = await fieldAttacher({ + projectId: 'proj-1', + orgId: 'org-1', + destinationStackId: 'stack-1', + region: 'NA', + user_id: 'user-1', + }); + + expect(mockShouldSkipContentTypeCreation).toHaveBeenCalledWith('proj-1', 'blog', 2); + expect(mockContenTypeMaker).not.toHaveBeenCalled(); + expect(result).toHaveLength(1); + }); }); diff --git a/api/tests/unit/utils/index.test.ts b/api/tests/unit/utils/index.test.ts index eefac615f..70caaf735 100644 --- a/api/tests/unit/utils/index.test.ts +++ b/api/tests/unit/utils/index.test.ts @@ -51,6 +51,14 @@ describe('utils/index', () => { it('should return false for boolean', () => { expect(isEmpty(false)).toBe(false); }); + + it('should return true for empty array', () => { + expect(isEmpty([])).toBe(true); + }); + + it('should return false for symbol', () => { + expect(isEmpty(Symbol('x'))).toBe(false); + }); }); describe('safePromise', () => { @@ -92,5 +100,15 @@ describe('utils/index', () => { const log = getLogMessage('testMethod', 'test message'); expect(log).not.toHaveProperty('error'); }); + + it('omits user spread when user is null', () => { + const log = getLogMessage('testMethod', 'test message', null as unknown as Record); + expect(log).not.toHaveProperty('user'); + }); + + it('omits error spread when error is falsy', () => { + const log = getLogMessage('testMethod', 'test message', {}, 0 as unknown as undefined); + expect(log).not.toHaveProperty('error'); + }); }); }); diff --git a/api/tests/unit/utils/pagination.utils.test.ts b/api/tests/unit/utils/pagination.utils.test.ts index 27ba63429..c729b088d 100644 --- a/api/tests/unit/utils/pagination.utils.test.ts +++ b/api/tests/unit/utils/pagination.utils.test.ts @@ -1,11 +1,18 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const { mockHttps } = vi.hoisted(() => ({ mockHttps: vi.fn() })); +const { mockHttps, mockSsoRequest } = vi.hoisted(() => ({ + mockHttps: vi.fn(), + mockSsoRequest: vi.fn(), +})); vi.mock('../../../src/utils/https.utils.js', () => ({ default: mockHttps, })); +vi.mock('../../../src/utils/sso-request.utils.js', () => ({ + requestWithSsoTokenRefresh: (...args: unknown[]) => mockSsoRequest(...args), +})); + vi.mock('../../../src/utils/index.js', async (importOriginal) => { const actual = await importOriginal(); return { @@ -20,6 +27,7 @@ import fetchAllPaginatedData from '../../../src/utils/pagination.utils.js'; describe('pagination.utils', () => { beforeEach(() => { vi.clearAllMocks(); + mockSsoRequest.mockResolvedValue([null, { data: { items: [] } }]); }); it('should fetch a single page of data', async () => { @@ -80,4 +88,21 @@ describe('pagination.utils', () => { fetchAllPaginatedData('https://api.example.com/data', {}, 100, 'testFunc', 'items') ).rejects.toThrow('is not iterable'); }); + + it('uses requestWithSsoTokenRefresh when is_sso token payload is passed', async () => { + mockSsoRequest.mockResolvedValue([null, { data: { items: [{ id: 'a' }] } }]); + + const result = await fetchAllPaginatedData( + 'https://api.example.com/data', + {}, + 100, + 'ssoFunc', + 'items', + { region: 'NA', user_id: 'u1', is_sso: true } + ); + + expect(mockSsoRequest).toHaveBeenCalled(); + expect(mockHttps).not.toHaveBeenCalled(); + expect(result).toEqual([{ id: 'a' }]); + }); }); diff --git a/api/tests/unit/utils/sanitize-path.utils.test.ts b/api/tests/unit/utils/sanitize-path.utils.test.ts index bb60825f7..2613dc368 100644 --- a/api/tests/unit/utils/sanitize-path.utils.test.ts +++ b/api/tests/unit/utils/sanitize-path.utils.test.ts @@ -1,5 +1,9 @@ -import { describe, it, expect } from 'vitest'; -import { sanitizeStackId, getSafePath } from '../../../src/utils/sanitize-path.utils.js'; +import { describe, it, expect, vi } from 'vitest'; +import { + sanitizeStackId, + getSafePath, + assertResolvedPathUnderBase, +} from '../../../src/utils/sanitize-path.utils.js'; import path from 'path'; describe('sanitize-path.utils', () => { @@ -86,5 +90,32 @@ describe('sanitize-path.utils', () => { expect(result).toContain('file.log'); expect(path.isAbsolute(result)).toBe(true); }); + + it('should return fallback when path resolution throws', () => { + const resolveSpy = vi.spyOn(path, 'resolve').mockImplementationOnce(() => { + throw new Error('resolve failed'); + }); + + const result = getSafePath('file.log', '/tmp/logs'); + + expect(result).toBe(path.join('/tmp/logs', 'default.log')); + resolveSpy.mockRestore(); + }); + }); + + describe('assertResolvedPathUnderBase', () => { + it('should not throw for paths inside base directory', () => { + expect(() => + assertResolvedPathUnderBase('/tmp/logs', '/tmp/logs/subdir/file.log') + ).not.toThrow(); + }); + + it('should throw for paths outside base directory', () => { + expect(() => + assertResolvedPathUnderBase('/tmp/logs', '/tmp/other/file.log') + ).toThrow( + 'Invalid path: resolved location is outside the allowed base directory' + ); + }); }); }); diff --git a/api/tests/unit/utils/sso-request.utils.test.ts b/api/tests/unit/utils/sso-request.utils.test.ts new file mode 100644 index 000000000..ce75e4662 --- /dev/null +++ b/api/tests/unit/utils/sso-request.utils.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { mockHttps, mockRefresh } = vi.hoisted(() => ({ + mockHttps: vi.fn(), + mockRefresh: vi.fn(), +})); + +vi.mock('../../../src/utils/https.utils.js', () => ({ default: mockHttps })); +vi.mock('../../../src/services/auth.service.js', () => ({ + refreshOAuthToken: mockRefresh, +})); +vi.mock('../../../src/utils/logger.js', () => ({ + default: { error: vi.fn(), info: vi.fn(), warn: vi.fn() }, +})); + +describe('sso-request.utils', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns first [err,res] when request succeeds', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + mockHttps.mockResolvedValueOnce({ status: 200, data: {} }); + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: true }, + { url: 'https://x', method: 'GET' } + ); + expect(out[0]).toBeNull(); + expect(out[1]?.status).toBe(200); + expect(mockRefresh).not.toHaveBeenCalled(); + }); + + it('returns [err,res] when not SSO without refresh', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + mockHttps.mockRejectedValueOnce({ response: { status: 401, data: { error_code: 105 } } }); + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: false }, + { url: 'https://x', method: 'GET' } + ); + expect(out[0]).toBeDefined(); + expect(mockRefresh).not.toHaveBeenCalled(); + }); + + it('refreshes token and retries on 401 for SSO', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + const err401 = { response: { status: 401, data: { error_code: 105 } } }; + mockHttps.mockRejectedValueOnce(err401).mockResolvedValueOnce({ status: 200, data: { ok: true } }); + mockRefresh.mockResolvedValue('new-access'); + + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: true }, + { url: 'https://x', method: 'GET', headers: {} } + ); + + expect(mockRefresh).toHaveBeenCalledWith('u1'); + expect(mockHttps).toHaveBeenCalledTimes(2); + expect(out[1]?.data?.ok).toBe(true); + }); + + it('returns original error when refresh throws', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + const err401 = { response: { status: 401, data: { code: 105 } } }; + mockHttps.mockRejectedValueOnce(err401); + mockRefresh.mockRejectedValue(new Error('refresh failed')); + + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: true }, + { url: 'https://x', method: 'GET' } + ); + expect(out[0]).toBe(err401); + }); + + it('refreshes on 401 even when error body has no code', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + mockHttps + .mockRejectedValueOnce({ response: { status: 401, data: {} } }) + .mockResolvedValueOnce({ status: 200, data: { ok: 1 } }); + mockRefresh.mockResolvedValue('tok2'); + + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: true }, + { url: 'https://x', method: 'GET' } + ); + expect(mockRefresh).toHaveBeenCalled(); + expect(out[1]?.data?.ok).toBe(1); + }); + + it('does not refresh when error is not a token error', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + const err403 = { response: { status: 403, data: {} } }; + mockHttps.mockRejectedValueOnce(err403); + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: true }, + { url: 'https://x', method: 'GET' } + ); + expect(out[0]).toBe(err403); + expect(mockRefresh).not.toHaveBeenCalled(); + }); + + it('refreshes when error_code is 105 even if HTTP status is not 401', async () => { + const { requestWithSsoTokenRefresh } = await import('../../../src/utils/sso-request.utils.js'); + mockHttps + .mockRejectedValueOnce({ response: { status: 500, data: { error_code: 105 } } }) + .mockResolvedValueOnce({ status: 200, data: { ok: 2 } }); + mockRefresh.mockResolvedValue('tok3'); + + const out = await requestWithSsoTokenRefresh( + { region: 'NA', user_id: 'u1', is_sso: true }, + { url: 'https://x', method: 'GET' } + ); + expect(mockRefresh).toHaveBeenCalled(); + expect(out[1]?.data?.ok).toBe(2); + }); +}); diff --git a/package.json b/package.json index df17c13ed..9eb149068 100644 --- a/package.json +++ b/package.json @@ -58,4 +58,4 @@ "@contentstack/cli-utilities": "^1.18.3", "qs": "^6.14.2" } -} +} \ No newline at end of file diff --git a/ui/package.json b/ui/package.json index 642ee3549..47ab83777 100644 --- a/ui/package.json +++ b/ui/package.json @@ -76,4 +76,4 @@ } } } -} +} \ No newline at end of file diff --git a/ui/src/components/ContentMapper/contentMapper.interface.ts b/ui/src/components/ContentMapper/contentMapper.interface.ts index 7b3b75024..23a52ac58 100644 --- a/ui/src/components/ContentMapper/contentMapper.interface.ts +++ b/ui/src/components/ContentMapper/contentMapper.interface.ts @@ -224,3 +224,16 @@ export interface ModifiedField { _canSelect?: boolean; contentstackFieldType?: string; } + +export interface EntryMapperType { + id: string; + projectId: string; + contentTypeId: string; + contentTypeUid: string; + entryName: string; + otherCmsEntryUid: string; + isUpdate: boolean; + contentstackEntryUid?: string; + _canSelect?: boolean; + isDuplicateEntry?: boolean; +} \ No newline at end of file diff --git a/ui/src/components/ContentMapper/entryMapper.tsx b/ui/src/components/ContentMapper/entryMapper.tsx new file mode 100644 index 000000000..29fa5f0bc --- /dev/null +++ b/ui/src/components/ContentMapper/entryMapper.tsx @@ -0,0 +1,425 @@ +// Libraries +import { useEffect, useState} from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { useParams } from 'react-router-dom'; +import { + Button, + InfiniteScrollTable, + Notification, + +} from '@contentstack/venus-components'; + +// Services +import { getCMSDataFromFile } from '../../cmsData/cmsSelector'; +import { + getContentTypes, + getEntryMapping, + updateEntryMapper, +} from '../../services/api/migration.service'; + +// Redux +import { RootState } from '../../store'; +import { updateMigrationData } from '../../store/slice/migrationDataSlice'; + +// Utilities +import { CS_ENTRIES } from '../../utilities/constants'; +import useBlockNavigation from '../../hooks/userNavigation'; + +// Interface +import { DEFAULT_CONTENT_MAPPING_DATA } from '../../context/app/app.interface'; +import { + ContentType, + TableTypes, + UidMap, + EntryMapperType +} from './contentMapper.interface'; +import { ItemStatusMapProp } from '@contentstack/venus-components/build/components/Table/types'; + + +// Styles and Assets +import './index.scss'; + +const EntryMapper = ({selectedContentTypeId, tableHeight}: {selectedContentTypeId: ContentType | null, tableHeight: number}) => { + // Redux State + const dispatch = useDispatch(); + + const { projectId = '' } = useParams<{ projectId: string }>(); + + const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + const selectedOrganisation = useSelector((state: RootState) => state?.authentication?.selectedOrganisation); + + // Component State + const [tableData, setTableData] = useState([]); + const [loading, setLoading] = useState(false); + const [totalCounts, setTotalCounts] = useState(tableData?.length); + const [searchText, setSearchText] = useState(''); + const [selectedContentType, setSelectedContentType] = useState(selectedContentTypeId); + const [contentTypes, setContentTypes] = useState([]); + const [itemStatusMap, setItemStatusMap] = useState({}); + + const [otherCmsTitle, setOtherCmsTitle] = useState(''); + const [contentTypeUid, setContentTypeUid] = useState(selectedContentTypeId?.id || ''); + + const [isContentType, setIsContentType] = useState(true); + + const [otherCmsUid, setOtherCmsUid] = useState(contentTypes?.[0]?.otherCmsUid); + const [rowIds, setRowIds] = useState>({}); + const [persistedRowIds, setPersistedRowIds] = useState>({}); + const [isLoadingSaveButton, setisLoadingSaveButton] = useState(false); + const [initialRowSelectedData, setInitialRowSelectedData] = useState([]); + + + + /********** ALL USEEFFECT HERE *************/ + useEffect(() => { + //check if offline CMS data field is set to true, if then read data from cms data file. + getCMSDataFromFile(CS_ENTRIES.CONTENT_MAPPING) + .then((data) => { + //Check for null + if (!data) { + dispatch(updateMigrationData({ contentMappingData: DEFAULT_CONTENT_MAPPING_DATA })); + return; + } + + dispatch(updateMigrationData({ contentMappingData: data })); + }) + .catch((err) => { + console.error(err); + }); + + fetchContentTypes(searchText || ''); + }, []); + + useEffect(() => { + if (selectedContentTypeId) { + fetchEntries(selectedContentTypeId?.id || '', searchText); + setOtherCmsTitle(selectedContentTypeId?.otherCmsTitle); + } + + },[selectedContentTypeId]); + + const buildSelectedRowIds = (entries: EntryMapperType[]) => { + return (entries ?? []).reduce((acc, item) => { + if (item?._canSelect && item?.isUpdate) { + acc[item.id] = true; + } + return acc; + }, {}); + }; + + const applySelectionToEntries = ( + entries: EntryMapperType[], + selected: Record, + ) => { + return (entries ?? []).map((item) => { + if (!item?._canSelect) return item; + return { + ...item, + isUpdate: !!selected?.[item.id], + }; + }); + }; + + const fetchContentTypes = async (searchText: string) => { + //setIsLoading(true); + + try { + const { data } = await getContentTypes(projectId || '', 0, 5000, ''); //org id will always present + + + setContentTypes(data?.contentTypes); + setSelectedContentType(data?.contentTypes?.[0]); + setOtherCmsTitle(data?.contentTypes?.[0]?.otherCmsTitle); + setContentTypeUid(data?.contentTypes?.[0]?.id); + fetchEntries(data?.contentTypes?.[0]?.id, searchText ?? ''); + setOtherCmsUid(data?.contentTypes?.[0]?.otherCmsUid); + setIsContentType(data?.contentTypes?.[0]?.type === "content_type"); + } catch (error) { + console.error(error); + return error; + } + }; + + // Method to get fieldmapping + const fetchEntries = async (contentTypeId: string, searchText: string) => { + try { + const itemStatusMap: ItemStatusMapProp = {}; + + for (let index = 0; index <= 1000; index++) { + itemStatusMap[index] = 'loading'; + } + + setItemStatusMap(itemStatusMap); + setLoading(true); + + const { data } = await getEntryMapping(contentTypeId || '', 0, 1000, searchText, projectId); + + for (let index = 0; index <= 1000; index++) { + itemStatusMap[index] = 'loaded'; + } + + setItemStatusMap({ ...itemStatusMap }); + setLoading(false); + + const validTableData: EntryMapperType[] = (data?.entryMapping ?? []).map((entry: EntryMapperType) => ({ + ...entry, + _canSelect: !!entry?.contentstackEntryUid, + })); + + //setIsAllCheck(true); + const initialSelected = buildSelectedRowIds(validTableData ?? []); + setTableData(validTableData ?? []); + setRowIds(initialSelected); + setPersistedRowIds(initialSelected); + setTotalCounts(validTableData?.length); + setInitialRowSelectedData(validTableData?.filter((item: EntryMapperType) => !item?.isUpdate)) + + } catch (error) { + console.error('fetchData -> error', error); + } + }; + + // Fetch table data + const fetchData = async ({ searchText }: TableTypes) => { + setSearchText(searchText) + selectedContentTypeId?.id && fetchEntries(selectedContentTypeId?.id, searchText); + }; + + // Method for Load more table data + const loadMoreItems = async ({ searchText, skip, limit, startIndex, stopIndex }: TableTypes) => { + try { + const itemStatusMapCopy: ItemStatusMapProp = { ...itemStatusMap }; + + for (let index = startIndex; index <= stopIndex; index++) { + itemStatusMapCopy[index] = 'loading'; + } + + setItemStatusMap({ ...itemStatusMapCopy }); + setLoading(true); + + const { data } = await getEntryMapping(contentTypeUid || '', skip, limit, searchText || '', projectId); + + const updateditemStatusMapCopy: ItemStatusMapProp = { ...itemStatusMap }; + + for (let index = startIndex; index <= stopIndex; index++) { + updateditemStatusMapCopy[index] = 'loaded'; + } + + setItemStatusMap({ ...updateditemStatusMapCopy }); + setLoading(false); + + const validTableData: EntryMapperType[] = (data?.entryMapping ?? []).map((entry: EntryMapperType) => ({ + ...entry, + _canSelect: !!entry?.contentstackEntryUid, + })); + + // eslint-disable-next-line no-unsafe-optional-chaining + setTableData(applySelectionToEntries(validTableData ?? [], rowIds)); + + } catch (error) { + console.error('loadMoreItems -> error', error); + } + }; + + /** + * Handle the selected entries + * @param singleSelectedRowIds - The single selected row IDs + * @returns void + */ + const handleSelectedEntries = (singleSelectedRowIds: string[]) => { + const selectedObj: UidMap = {}; + singleSelectedRowIds?.forEach((uid: string) => { + selectedObj[uid] = true; + }); + + setRowIds(selectedObj); + setTableData((prev) => applySelectionToEntries(prev ?? [], selectedObj)); + }; + + const handleSaveContentType = async () => { + console.info("handleSaveContentType", rowIds); + setisLoadingSaveButton(true); + const allKeys = new Set([ + ...Object.keys(rowIds ?? {}), + ...Object.keys(persistedRowIds ?? {}), + ]); + const changedUids = Array.from(allKeys).filter( + (uid) => !!rowIds?.[uid] !== !!persistedRowIds?.[uid], + ); + const orgId = selectedOrganisation?.uid; + // const projectID = projectId; + + if (orgId && contentTypeUid) { + const dataCs = { + ids: changedUids + }; + try { + if (changedUids.length === 0) { + setisLoadingSaveButton(false); + return Notification({ + notificationContent: { text: 'No changes to save' }, + notificationProps: { + position: 'bottom-center', + hideProgressBar: true + }, + type: 'info' + }); + } + const {data, status} = await updateEntryMapper(projectId, dataCs); + console.info("status", status, typeof status, data); + + setisLoadingSaveButton(false); + if (status === 200) { + setPersistedRowIds({ ...(rowIds ?? {}) }); + setLoading(false); + return Notification({ + notificationContent: { text: 'Entries saved successfully' }, + notificationProps: { + position: 'bottom-center', + hideProgressBar: true + }, + type: 'success' + }); + } + else{ + setisLoadingSaveButton(false); + return Notification({ + notificationContent: { text: 'Failed to save entries' }, + notificationProps: { + position: 'bottom-center', + hideProgressBar: true + }, + type: 'error' + }); + } + } catch (error) { + console.error(error); + setisLoadingSaveButton(false); + return error; + } + + } + } + const accessorCall = (data: EntryMapperType) => { + // Clean field name (remove parent hierarchy) + const cleanFieldName = data?.entryName + return ( +
+
+
+ {cleanFieldName} +
+
+
+ ); + }; + + const accessorContentstackCall = (data: EntryMapperType) => { + // Clean field name (remove parent hierarchy) + const cleanFieldName = data?.contentstackEntryUid + return ( +
+
+
+ {cleanFieldName ? cleanFieldName : '-'} +
+
+
+ + ); + }; + + const accessorForCMSUid = (data: EntryMapperType) => { + const cleanFieldName = data?.otherCmsEntryUid + return ( +
+
+
+ {cleanFieldName ? cleanFieldName : '-'} +
+
+
+ ); + } + + const columns = [ + { + disableSortBy: true, + Header: ( + + {`${newMigrationData?.legacy_cms?.selectedCms?.title}: ${otherCmsTitle}`} + + ), + accessor: accessorCall, + id: 'uuid', + width: '250px', + }, + { + disableSortBy: true, + Header: ( + + {`${newMigrationData?.legacy_cms?.selectedCms?.title} UIDs:`} + + ), + accessor: accessorForCMSUid, + id: '1' + }, + { + disableSortBy: true, + Header: ( + + {'Contentstack UIDs:'} + + ), + accessor: accessorContentstackCall, + id: '2' + } + ]; + + return ( +
+ +
+
Total Entries: {totalCounts}
+ +
+ + +
+ + ) +} +export default EntryMapper; \ No newline at end of file diff --git a/ui/src/components/ContentMapper/index.scss b/ui/src/components/ContentMapper/index.scss index 17bc09b96..de99218de 100644 --- a/ui/src/components/ContentMapper/index.scss +++ b/ui/src/components/ContentMapper/index.scss @@ -140,56 +140,58 @@ .table-container { flex: 1 0 auto; } -.table-wrapper { - flex: 1; - .TablePanel { - border-left: 0 none; - .TablePanel__list-count { - display: none; +.content-mapper-container { + .table-wrapper { + flex: 1; + .TablePanel { + border-left: 0 none; + .TablePanel__list-count { + display: none; + } } - } - .Table { - border-left: 0 none; - min-height: 24.25rem; - .Table__body__row { + .Table { + border-left: 0 none; + min-height: 24.25rem; + .Table__body__row { .Table-select-body { - >.checkbox-wrapper { - align-items: flex-start; + > .checkbox-wrapper { + align-items: flex-start; + } } } - } - .Table__body__column { - padding: 0 1.25rem; - &:not(:last-of-type) { - display: block; + .Table__body__column { + padding: 0 1.25rem; + &:not(:last-of-type) { + display: block; + } } - } - .Table-select-body { - width: 68px; - } - .cms-field { - // text-transform: capitalize; - // overflow: hidden; - // text-overflow: ellipsis; - // white-space: nowrap; - // width: 445px; + .Table-select-body { + width: 68px; + } + .cms-field { + // text-transform: capitalize; + // overflow: hidden; + // text-overflow: ellipsis; + // white-space: nowrap; + // width: 445px; - display: -webkit-box; - -webkit-box-orient: vertical; - -webkit-line-clamp: 1; - overflow: hidden; - text-overflow: ellipsis; + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 1; + overflow: hidden; + text-overflow: ellipsis; + } + .InstructionText { + font-size: $size-font-small; + margin: 0; + } + .EmptyStateWrapper { + margin-top: $px-20; + } } - .InstructionText { - font-size: $size-font-small; + .import-cta { margin: 0; } - .EmptyStateWrapper { - margin-top: $px-20; - } - } - .import-cta { - margin: 0; } } .disabled-field { diff --git a/ui/src/components/ContentMapper/index.tsx b/ui/src/components/ContentMapper/index.tsx index 67872a9b3..3d51293a0 100644 --- a/ui/src/components/ContentMapper/index.tsx +++ b/ui/src/components/ContentMapper/index.tsx @@ -85,6 +85,7 @@ import { // Styles and Assets import './index.scss'; import { NoDataFound, SCHEMA_PREVIEW } from '../../common/assets'; +import EntryMapper from './entryMapper'; const FIELD_MAP_MENU_VIEW_MARGIN = 8; const FIELD_MAP_MENU_HYSTERESIS = 36; @@ -493,7 +494,9 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: const migrationData = useSelector((state: RootState) => state?.migration?.migrationData); const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); const selectedOrganisation = useSelector((state: RootState) => state?.authentication?.selectedOrganisation); - + const iteration = useSelector( + (state: RootState) => state?.migration?.newMigrationData?.iteration + ); // When setting contentModels from Redux, ensure it's cloned const reduxContentTypes = newMigrationData?.content_mapping?.existingCT; // Assume this gets your Redux state const reduxGlobalFields = newMigrationData?.content_mapping?.existingGlobal @@ -561,7 +564,12 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: const [activeFilter, setActiveFilter] = useState(''); const [isAllCheck, setIsAllCheck] = useState(false); const [isResetFetch, setIsResetFetch] = useState(false); + const [iterationCount, setIterationCount] = useState(newMigrationData?.iteration); + /** After reset-to-initial-mapping, do not re-apply UID auto-match until user picks a destination again. */ + const [uidAutoMapSuppressedForSourceUids, setUidAutoMapSuppressedForSourceUids] = useState>( + () => new Set() + ); /** ALL HOOKS Here */ const { projectId = '' } = useParams(); @@ -592,6 +600,14 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: fetchContentTypes(searchText || ''); }, []); + useEffect(() => { + const currentIteration = newMigrationData?.iteration || 1; + if (currentIteration !== iterationCount) { + setIterationCount(currentIteration); + fetchContentTypes(searchText || ''); + } + }, [newMigrationData?.iteration, iterationCount, searchText]); + // Make title and url field non editable useEffect(() => { tableData?.forEach((field) => { @@ -3174,7 +3190,7 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: ([sourceUid, mappedDestUid]) => mappedDestUid === destinationUid && sourceUid !== selectedContentType?.contentstackUid ); - + const sourceContentTypeUids = new Set( (contentTypes ?? []) .map((ct) => ct?.contentstackUid) @@ -3380,6 +3396,15 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: {/* Content Type Fields */}
+ {iteration > 1 ? ( +
+ +
+ ): ( +
+ )} +
: { const isSQL = fileFormatId?.toLowerCase() === 'sql'; + const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + const [isEditing, setIsEditing] = useState((newMigrationData?.iteration > 1 && !newMigrationData?.legacy_cms?.uploadedFile?.isValidated) ? true : false); + const [localPath, setLocalPath] = useState(fileDetails?.localPath || ''); + const dispatch = useDispatch(); + const currentPath = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath || fileDetails?.localPath || ''; + const iteration = newMigrationData?.iteration || 1; + + const handleEditFile = async () => { + setIsEditing(true); + setLocalPath(currentPath); + }; + + const handleBlur = async () => { + setIsEditing(false); + + // Update Redux state with new path + const updatedMigrationData = { + ...newMigrationData, + legacy_cms: { + ...newMigrationData?.legacy_cms, + uploadedFile: { + ...newMigrationData?.legacy_cms?.uploadedFile, + name: localPath, + url: localPath, + file_details: { + ...newMigrationData?.legacy_cms?.uploadedFile?.file_details, + localPath: localPath + } + } + } + }; + dispatch(updateNewMigrationData(updatedMigrationData)); + }; + return (
@@ -67,8 +101,28 @@ const FileComponent = ( { fileDetails, fileFormatId }: Props ) => ) : fileDetails?.isLocalPath ? ( // ✅ Local path (file or directory — format driven by legacyCms.json)
- +
+ {isEditing ? ( + ) => setLocalPath(e.target.value)} + onBlur={handleBlur} + width="full" + version="v2" + placeholder="Enter local path" + aria-label="local path" + autoFocus + /> + ) : ( + + )}
+ { iteration > 1 && ( +
+ +
+ )} +
) : ( // ✅ AWS S3 details (isLocalPath is false)
diff --git a/ui/src/components/LegacyCms/legacyCms.scss b/ui/src/components/LegacyCms/legacyCms.scss index 550ddb95d..1b460251c 100644 --- a/ui/src/components/LegacyCms/legacyCms.scss +++ b/ui/src/components/LegacyCms/legacyCms.scss @@ -156,6 +156,9 @@ margin-top: 5px; } .file-container { + display: flex; + justify-content: space-between; + align-items: center; overflow: hidden; padding: 0; white-space: normal; @@ -163,6 +166,30 @@ line-clamp: 2; -webkit-box-orient: vertical; width: 540px; + gap: 10px; + + .file-path-text { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + + .TextInput { + width: 100%; + } + } + + .edit-icon { + flex-shrink: 0; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + padding: 5px; + + &:hover { + opacity: 0.7; + } + } } .message-container { display: inline-flex; diff --git a/ui/src/components/LogScreen/MigrationLogViewer.tsx b/ui/src/components/LogScreen/MigrationLogViewer.tsx index 0eea6be8b..eba1dccf8 100644 --- a/ui/src/components/LogScreen/MigrationLogViewer.tsx +++ b/ui/src/components/LogScreen/MigrationLogViewer.tsx @@ -52,6 +52,7 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { ]); const [isModalOpen, setIsModalOpen] = useState(false); const [zoomLevel, setZoomLevel] = useState(1); + const [hasShownCompletionNotification, setHasShownCompletionNotification] = useState(false); const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); const selectedOrganisation = useSelector( @@ -116,7 +117,21 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { }, []); useBlockNavigation(isModalOpen); + + useEffect(() => { + if (newMigrationData?.migration_execution?.migrationCompleted) { + dispatch(updateNewMigrationData({ stepValue: 'Restart Migration' })); + } + }, [newMigrationData?.migration_execution?.migrationCompleted, dispatch]); + + // Reset notification flag when a new migration starts + useEffect(() => { + if (newMigrationData?.migration_execution?.migrationStarted && !newMigrationData?.migration_execution?.migrationCompleted) { + setHasShownCompletionNotification(false); + } + }, [newMigrationData?.migration_execution?.migrationStarted, newMigrationData?.migration_execution?.migrationCompleted]); + /** * Scrolls to the top of the logs container. */ @@ -180,8 +195,9 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { //const logObject = JSON.parse(log); const message = log.message; - if (message === 'Migration Process Completed') { + if (message === 'Migration Process Completed' && !hasShownCompletionNotification) { setIsModalOpen(true); + setHasShownCompletionNotification(true); const newMigrationDataObj: INewMigration = { ...newMigrationData, @@ -189,7 +205,8 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { ...newMigrationData?.migration_execution, migrationStarted: false, migrationCompleted: true - } + }, + stepValue: 'Restart Migration' }; dispatch(updateNewMigrationData(newMigrationDataObj)); diff --git a/ui/src/components/MigrationFlowHeader/index.tsx b/ui/src/components/MigrationFlowHeader/index.tsx index 196c35ad9..b091ff1ef 100644 --- a/ui/src/components/MigrationFlowHeader/index.tsx +++ b/ui/src/components/MigrationFlowHeader/index.tsx @@ -1,7 +1,7 @@ // Libraries import { useEffect, useState } from 'react'; import { Button, Tooltip } from '@contentstack/venus-components'; -import { useSelector } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { Params, useNavigate, useParams } from 'react-router'; import { RootState } from '../../store'; @@ -11,6 +11,7 @@ import { MigrationResponse } from '../../services/api/service.interface'; // CSS import './index.scss'; +import { updateNewMigrationData } from '../../store/slice/migrationDataSlice'; type MigrationFlowHeaderProps = { handleOnClick: (event: MouseEvent, handleStepChange: (currentStep: number) => void) => void; @@ -44,6 +45,7 @@ const MigrationFlowHeader = ({ (state: RootState) => state?.authentication?.selectedOrganisation ); const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + const dispatch = useDispatch(); useEffect(() => { fetchProject(); @@ -62,14 +64,25 @@ const MigrationFlowHeader = ({ navigate(url, { replace: true }); }; - let stepValue; - if (params?.stepId === '3' || params?.stepId === '4') { - stepValue = 'Continue'; - } else if (params?.stepId === '5') { - stepValue = 'Start Migration'; - } else { - stepValue = 'Save and Continue'; - } + useEffect(() => { + let newStepValue; + + // Check conditions in priority order + if (newMigrationData?.legacy_cms?.projectStatus === 5 && newMigrationData?.migration_execution?.migrationCompleted) { + newStepValue = 'Restart Migration'; + } else if (params?.stepId === '5') { + newStepValue = 'Start Migration'; + } else if (params?.stepId === '3' || params?.stepId === '4') { + newStepValue = 'Continue'; + } else { + newStepValue = 'Save and Continue'; + } + + // Only update if the value has changed + if (newStepValue !== newMigrationData?.stepValue) { + dispatch(updateNewMigrationData({ stepValue: newStepValue })); + } + }, [params?.stepId, newMigrationData?.legacy_cms?.projectStatus, newMigrationData?.migration_execution?.migrationCompleted, newMigrationData?.stepValue, dispatch]); const isStep4AndNotMigrated = params?.stepId === '4' && @@ -129,13 +142,11 @@ const MigrationFlowHeader = ({ isProjectStatusThreeAndMapperNotGenerated ? isFileValidated : isStep4AndNotMigrated || - isStepInvalid || - isExecutionStarted || - destinationStackMigrated + isStepInvalid } > - {stepValue} - + {newMigrationData?.stepValue || 'Save and Continue'} +
); }; diff --git a/ui/src/components/Stepper/HorizontalStepper/HorizontalStepper.tsx b/ui/src/components/Stepper/HorizontalStepper/HorizontalStepper.tsx index 3a1be8f44..809c659e5 100644 --- a/ui/src/components/Stepper/HorizontalStepper/HorizontalStepper.tsx +++ b/ui/src/components/Stepper/HorizontalStepper/HorizontalStepper.tsx @@ -83,9 +83,15 @@ const HorizontalStepper = forwardRef( const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); const { steps, className, emptyStateMsg, hideTabView, testId } = props; - const [showStep, setShowStep] = useState(stepIndex); + + // Initialize showStep based on current state - if restarted, start from 0 + const initialStep = (newMigrationData?.project_current_step === 1 && + newMigrationData?.legacy_cms?.projectStatus === 0) ? 0 : stepIndex; + + const [showStep, setShowStep] = useState(initialStep); const [stepsCompleted, setStepsCompleted] = useState([]); const [isModalOpen, setIsModalOpen] = useState(false); + const [lastIteration, setLastIteration] = useState(newMigrationData?.iteration || 1); const navigate = useNavigate(); const { projectId = '' } = useParams(); @@ -99,31 +105,75 @@ const HorizontalStepper = forwardRef( newMigrationDataRef.current = newMigrationData; }, [newMigrationData]); + // Reset stepper when migration is restarted (detected by iteration increment) + useEffect(() => { + const currentIteration = newMigrationData?.iteration || 1; + + // If iteration has increased, it means migration was restarted + if (currentIteration > lastIteration) { + setStepsCompleted([]); + setShowStep(0); + setLastIteration(currentIteration); + } + }, [newMigrationData?.iteration, lastIteration]); + + // Also reset stepper when migration is restarted (fallback detection) + useEffect(() => { + // Check if migration was restarted by looking at multiple indicators + const isRestarted = + newMigrationData?.project_current_step === 1 && + newMigrationData?.legacy_cms?.projectStatus === 0 && + newMigrationData?.legacy_cms?.currentStep === 1 && + !newMigrationData?.migration_execution?.migrationCompleted && + !newMigrationData?.migration_execution?.migrationStarted; + + if (isRestarted && stepsCompleted?.length > 0) { + setStepsCompleted([]); + setShowStep(0); + } + }, [ + newMigrationData?.project_current_step, + newMigrationData?.legacy_cms?.projectStatus, + newMigrationData?.legacy_cms?.currentStep, + newMigrationData?.migration_execution?.migrationCompleted, + newMigrationData?.migration_execution?.migrationStarted, + stepsCompleted?.length + ]); + useEffect(() => { const stepIndex = parseInt(stepId || '', 10) - 1; if (!Number.isNaN(stepIndex) && stepIndex >= 0 && stepIndex < steps?.length) { !newMigrationDataRef?.current?.isprojectMapped && setShowStep(stepIndex); - setStepsCompleted((prev) => { - const updatedStepsCompleted = [...prev]; - if ( - stepIndex === 4 && - (props?.projectData?.isMigrationCompleted || - newMigrationData?.migration_execution?.migrationCompleted) - ) { - if (!updatedStepsCompleted?.includes(4)) { - updatedStepsCompleted.push(4); + + // Only auto-complete previous steps if migration hasn't been restarted recently + // Check if this is a fresh restart (project_current_step = 1 and low projectStatus) + const isFreshRestart = + newMigrationData?.project_current_step === 1 && + newMigrationData?.legacy_cms?.projectStatus === 0; + + if (!isFreshRestart) { + setStepsCompleted((prev) => { + const updatedStepsCompleted = [...prev]; + if ( + stepIndex === 4 && + (props?.projectData?.isMigrationCompleted || + newMigrationData?.migration_execution?.migrationCompleted) + ) { + if (!updatedStepsCompleted?.includes(4)) { + updatedStepsCompleted.push(4); + } } - } - for (let i = 0; i < stepIndex; i++) { - if (!updatedStepsCompleted?.includes(i)) { - updatedStepsCompleted?.push(i); + for (let i = 0; i < stepIndex; i++) { + if (!updatedStepsCompleted?.includes(i)) { + updatedStepsCompleted?.push(i); + } } - } - return updatedStepsCompleted; - }); + return updatedStepsCompleted; + }); + } } - }, [stepId, newMigrationData?.migration_execution?.migrationCompleted]); + }, [stepId, newMigrationData?.migration_execution?.migrationCompleted, newMigrationData?.project_current_step, newMigrationData?.legacy_cms?.projectStatus]); useImperativeHandle(ref, () => ({ handleStepChange: (currentStep: number) => { diff --git a/ui/src/context/app/app.interface.ts b/ui/src/context/app/app.interface.ts index 80ef579fd..dc34b0c00 100644 --- a/ui/src/context/app/app.interface.ts +++ b/ui/src/context/app/app.interface.ts @@ -216,6 +216,8 @@ export interface INewMigration { migration_execution: IMigrationExecutionStep; project_current_step: number; settings:ISetting; + iteration: number; + stepValue?: string; } export interface TestStacks { @@ -405,6 +407,8 @@ export const DEFAULT_NEW_MIGRATION: INewMigration = { project_current_step: 0, settings: DEFAULT_SETTING, isContentMapperGenerated: false, + iteration: 1, + stepValue: 'Save and Continue', }; export const DEFAULT_URL_TYPE: IURLType = { diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index 7dc709045..6e8fecda4 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -23,7 +23,8 @@ import { getExistingGlobalFields, startMigration, updateMigrationKey, - updateLocaleMapper + updateLocaleMapper, + restartMigration } from '../../services/api/migration.service'; import { getCMSDataFromFile } from '../../cmsData/cmsSelector'; @@ -808,6 +809,7 @@ const Migration = () => { const handleOnClickMigrationExecution = async () => { setIsLoading(true); + if (newMigrationData?.stepValue !== 'Restart Migration') { try { const migrationRes = await startMigration( newMigrationData?.destination_stack?.selectedOrg?.value, @@ -838,9 +840,48 @@ const Migration = () => { } catch (error) { // return error; console.error(error); + }} + else{ + setIsLoading(false); + handleRestartMigration(); } }; + const handleRestartMigration = async () => { + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + legacy_cms: { + ...newMigrationData?.legacy_cms, + projectStatus: 0, + currentStep: 1, + uploadedFile: { + ...newMigrationData?.legacy_cms?.uploadedFile, + isValidated: false + } + }, + migration_execution: { + ...newMigrationData?.migration_execution, + migrationStarted: false + }, + project_current_step: 1, + iteration: newMigrationData?.iteration ? newMigrationData?.iteration + 1 : 1 + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + const res = await restartMigration(selectedOrganisation?.value, projectId); + if (res?.status === 200) { + Notification({ + notificationContent: { text: 'Migration restarted successfully' }, + type: 'success' + }); + navigate(`/projects/${projectId}/migration/steps/1`); + } else { + Notification({ + notificationContent: { text: 'Failed to restart migration' }, + type: 'error' + }); + } + }; + /** * Once Save Changes Modal is shown, Change the dropdown state to false and store in rdux */ diff --git a/ui/src/services/api/migration.service.ts b/ui/src/services/api/migration.service.ts index 0555ccacc..27d662265 100644 --- a/ui/src/services/api/migration.service.ts +++ b/ui/src/services/api/migration.service.ts @@ -367,3 +367,53 @@ export const getMigrationLogs = async (orgId: string, projectId: string, stackId } } } + +export const restartMigration = async (orgId: string, projectId: string) => { + try { + return await postCall( + `${API_VERSION}/migration/restart/${orgId}/${projectId}`, {}, options()); + } catch (error) { + return error; + } +} + +export const getEntryMapping = async ( + contentTypeId: string, + skip: number, + limit: number, + searchText: string, + projectId: string +) => { + try { + const encodedSearchText = encodeURIComponent(searchText); + return await getCall( + `${API_VERSION}/mapper/entryMapping/${projectId}/${contentTypeId}/${skip}/${limit}/${encodedSearchText}?`, + options() + ); + } catch (error) { + if (error instanceof Error) { + throw new Error(`${error.message}`); + } else { + throw new Error('Unknown error'); + } + } +}; +export const updateEntryMapper = async ( + projectId: string, + data: ObjectType +) => { + try { + return await putCall( + `${API_VERSION}/mapper/updateEntryStatus/${projectId}`, + data, + options() + ); + } catch (error) { + if (error instanceof Error) { + throw new Error(`${error.message}`); + } else { + throw new Error('Unknown error'); + } + } +}; + diff --git a/ui/tests/unit/utilities/constants.test.ts b/ui/tests/unit/utilities/constants.test.ts index ee2c057f0..7df7686bf 100644 --- a/ui/tests/unit/utilities/constants.test.ts +++ b/ui/tests/unit/utilities/constants.test.ts @@ -118,6 +118,7 @@ describe('utilities/constants', () => { expect(STATUS_ICON_Mapping['1']).toBe('CheckedCircle'); expect(STATUS_ICON_Mapping['2']).toBe('SuccessInverted'); expect(STATUS_ICON_Mapping['3']).toBe('ErrorInverted'); + expect(Object.keys(STATUS_ICON_Mapping).sort()).toEqual(['1', '2', '3']); }); it('should export VALIDATION_DOCUMENTATION_URL', () => { diff --git a/upload-api/migration-contentful/index.js b/upload-api/migration-contentful/index.js index d38b50625..babec0d9b 100644 --- a/upload-api/migration-contentful/index.js +++ b/upload-api/migration-contentful/index.js @@ -4,10 +4,12 @@ const extractContentTypes = require('./libs/extractContentTypes'); const createInitialMapper = require('./libs/createInitialMapper'); const extractLocale = require('./libs/extractLocale'); const extractTaxonomy = require('./libs/extractTaxonomy'); +const extractEntries = require('./libs/extractEntries'); module.exports = { extractContentTypes, createInitialMapper, extractLocale, - extractTaxonomy + extractTaxonomy, + extractEntries }; diff --git a/upload-api/migration-contentful/libs/createInitialMapper.js b/upload-api/migration-contentful/libs/createInitialMapper.js index 3a3cdc761..e0df3fe41 100644 --- a/upload-api/migration-contentful/libs/createInitialMapper.js +++ b/upload-api/migration-contentful/libs/createInitialMapper.js @@ -8,6 +8,7 @@ const fs = require('fs/promises'); const path = require('path'); // const contentTypeMapper = require('./contentTypeMapper'); const contentTypeMapper = require('./contentTypeMapper'); +const extractEntries = require('./extractEntries'); /** Contentstack taxonomy_uid: lowercase, a-z0-9_ only */ function contentfulSchemeIdToStackTaxonomyUid(contentfulSchemeId) { @@ -121,6 +122,8 @@ const createInitialMapper = async (cleanLocalPath, affix) => { ctMetaById[ct.sys.id] = ct.metadata || {}; } } + + const entriesByContentType = extractEntries(cleanLocalPath); const initialMapper = []; const files = await fs.readdir( @@ -129,9 +132,10 @@ const createInitialMapper = async (cleanLocalPath, affix) => { for (const file of files) { const data = readFile( - path.resolve(process.cwd(), `${config.data}/${config.contentful.contentful}/${file}`) + path.resolve(process.cwd(), `${config?.data}/${config?.contentful?.contentful}/${file}`) ); const title = file.split('.')[0]; + const contentfulID = data?.[0]?.contentfulID; const contentTypeObject = { status: 1, @@ -142,7 +146,8 @@ const createInitialMapper = async (cleanLocalPath, affix) => { contentstackTitle: title.charAt(0).toUpperCase() + title.slice(1), contentstackUid: uidCorrector(data?.[0]?.contentUid, affix), type: 'content_type', - fieldMapping: [] + fieldMapping: [], + entryMapping: entriesByContentType[contentfulID] || [] }; const uidTitle = [ { @@ -185,4 +190,4 @@ const createInitialMapper = async (cleanLocalPath, affix) => { } }; -module.exports = createInitialMapper; +module.exports = createInitialMapper; \ No newline at end of file diff --git a/upload-api/migration-contentful/libs/extractEntries.js b/upload-api/migration-contentful/libs/extractEntries.js new file mode 100644 index 000000000..3e9529ed8 --- /dev/null +++ b/upload-api/migration-contentful/libs/extractEntries.js @@ -0,0 +1,57 @@ +'use strict'; +/* eslint-disable @typescript-eslint/no-var-requires */ + +const { readFile } = require('../utils/helper'); + +/** + * Extracts entries from a Contentful export and groups them by content type ID. + * + * @param {string} cleanLocalPath - Path to the Contentful export JSON file. + * @returns {Record} A map of contentTypeId to array of entry mapping objects. + */ +const extractEntries = (cleanLocalPath) => { + try { + const alldata = readFile(cleanLocalPath); + const { entries } = alldata; + const locales = alldata?.locales?.map((locale) => locale?.code); + + if (!entries || !Array.isArray(entries) || entries.length === 0) { + console.info('No entries found in Contentful export'); + return {}; + } + + const entriesByContentType = {}; + + for (const entry of entries) { + const contentTypeId = entry?.sys?.contentType?.sys?.id; + const entryId = entry?.sys?.id; + for (const locale of locales) { + let entryTitle = entry?.fields?.title?.[locale]; + entryTitle = !entryTitle ? entry?.fields?.name?.[locale] : entryTitle; + if (!entryTitle) continue; + if (!entriesByContentType[contentTypeId]) { + entriesByContentType[contentTypeId] = []; + } + + entriesByContentType[contentTypeId].push({ + contentTypeUid: contentTypeId, + entryName: entryTitle, + otherCmsEntryUid: entryId, + isUpdate: false, + language: locale, + }); + } + } + + console.info( + `extractEntries: Extracted entries for ${Object.keys(entriesByContentType).length} content types` + ); + + return entriesByContentType; + } catch (err) { + console.error('Error extracting Contentful entries:', err); + return {}; + } +}; + +module.exports = extractEntries; diff --git a/upload-api/migration-drupal/index.js b/upload-api/migration-drupal/index.js index ac588e200..ec551137e 100644 --- a/upload-api/migration-drupal/index.js +++ b/upload-api/migration-drupal/index.js @@ -3,10 +3,12 @@ const extractTaxonomy = require('./libs/extractTaxonomy'); const createInitialMapper = require('./libs/createInitialMapper'); const extractLocale = require('./libs/extractLocale'); +const extractEntries = require('./libs/extractEntries'); module.exports = { // extractContentTypes, extractTaxonomy, createInitialMapper, - extractLocale + extractLocale, + extractEntries }; diff --git a/upload-api/migration-drupal/libs/createInitialMapper.js b/upload-api/migration-drupal/libs/createInitialMapper.js index cbcd41242..4bb0e2664 100644 --- a/upload-api/migration-drupal/libs/createInitialMapper.js +++ b/upload-api/migration-drupal/libs/createInitialMapper.js @@ -239,6 +239,7 @@ const createInitialMapper = async (systemConfig, prefix) => { (contentType) => contentType && contentType.toLowerCase() !== 'profile' ); + const entryMapping = await extractEntries(connection, prefix); // Process each content type for (const contentType of contentTypes) { // Extra safety check - skip if contentType is profile (case-insensitive) @@ -263,7 +264,8 @@ const createInitialMapper = async (systemConfig, prefix) => { contentstackTitle: contenttypeTitle.charAt(0).toUpperCase() + contenttypeTitle.slice(1), contentstackUid: uidCorrector(contenttypeTitle, prefix), type: 'content_type', - fieldMapping: [] + fieldMapping: [], + entryMapping: entryMapping[contentType] || [] }; // Map fields using contentTypeMapper, passing actual taxonomy usage diff --git a/upload-api/migration-drupal/libs/extractEntries.js b/upload-api/migration-drupal/libs/extractEntries.js new file mode 100644 index 000000000..66ab2365e --- /dev/null +++ b/upload-api/migration-drupal/libs/extractEntries.js @@ -0,0 +1,104 @@ +// upload-api/migration-drupal/libs/extractEntries.js + +'use strict'; +/* eslint-disable @typescript-eslint/no-var-requires */ + +/** + * Builds per-bundle entry lists for the content mapper (entryMapping). + * (uidCorrector + `content_type_entries_title_${nid}`) so iteration > 1 + * updates (removeEntriesFromDatabase + entry-update-script) line up with locale JSON keys. + */ + +const DEFAULT_PREFIX = 'cs'; + +function startsWithNumber(str) { + return typeof str === 'string' && /^\d/.test(str); +} + +/** createMapper may pass affix as string | string[] */ +function normalizeAffix(prefix) { + if (prefix == null) { + return ''; + } + if (Array.isArray(prefix)) { + const first = prefix[0]; + return first != null && String(first).trim() !== '' ? String(first) : ''; + } + const s = String(prefix); + return s.trim() !== '' ? s : ''; +} + +/** + * `uidCorrector` for the `id` branch (no separate `uid`). + */ +function entrySourceUidCorrector({ id, prefix }) { + const value = id != null && id !== '' ? String(id) : ''; + if (!value) { + return ''; + } + const affix = normalizeAffix(prefix); + const effectivePrefix = affix !== '' ? affix : DEFAULT_PREFIX; + if (startsWithNumber(value)) { + return `${effectivePrefix}_${value.replace(/[ -]/g, '_').toLowerCase()}`; + } + return value.replace(/[ -]/g, '_').toLowerCase(); +} + +/** + * @param {import('mysql2').Connection} connection - Open mysql2 connection + * @param {string} [prefix] - Stack affix / prefix + * @returns {Promise>>} Map of Drupal bundle machine name - entryMapping rows + */ +async function extractEntries(connection, prefix) { + const byBundle = {}; + if (!connection || typeof connection.promise !== 'function') { + console.warn('extractEntries (Drupal): invalid connection, skipping'); + return byBundle; + } + + try { + const query = ` + SELECT nid, title, langcode, type + FROM node_field_data + WHERE status = 1 + AND LOWER(type) <> 'profile' + ORDER BY type, nid, langcode + `; + const [rows] = await connection.promise().query(query); + + for (const row of rows) { + const bundle = row?.type; + if (!bundle) { + continue; + } + const otherCmsEntryUid = entrySourceUidCorrector({ + id: `content_type_entries_title_${row?.nid}`, + prefix, + }); + const entryName = row?.title ? String(row?.title) : `Node ${row?.nid}`; + + if (!byBundle[bundle]) { + byBundle[bundle] = []; + } + byBundle[bundle].push({ + contentTypeUid: bundle, + entryName, + language: row?.langcode || '', + otherCmsEntryUid, + otherCmsCTName: bundle, + isUpdate: false, + }); + } + + const total = Object?.values(byBundle)?.reduce((n, arr) => n + arr?.length, 0); + console.info( + `extractEntries (Drupal): ${total} entries across ${Object.keys(byBundle).length} bundle(s)` + ); + return byBundle; + } catch (err) { + console.error('extractEntries (Drupal) error:', err?.message || err); + return byBundle; + } +} + +module.exports = extractEntries; \ No newline at end of file diff --git a/upload-api/migration-sitecore/index.js b/upload-api/migration-sitecore/index.js index ce84e6dae..2c80c5120 100644 --- a/upload-api/migration-sitecore/index.js +++ b/upload-api/migration-sitecore/index.js @@ -9,11 +9,12 @@ const reference = require('./libs/reference.js'); const ExtractFiles = require('./libs/convert.js'); // eslint-disable-next-line @typescript-eslint/no-var-requires const extractLocales = require('./libs/extractLocales.js'); - +const extractEntries = require('./libs/extractEntries.js'); module.exports = { contentTypes, ExtractConfiguration, reference, ExtractFiles, - extractLocales + extractLocales, + extractEntries }; diff --git a/upload-api/migration-sitecore/libs/extractEntries.js b/upload-api/migration-sitecore/libs/extractEntries.js new file mode 100644 index 000000000..450f1de13 --- /dev/null +++ b/upload-api/migration-sitecore/libs/extractEntries.js @@ -0,0 +1,211 @@ +const path = require('path'); +const fs = require('fs'); +const _ = require('lodash'); +const read = require('fs-readdir-recursive'); +const helper = require('../utils/helper'); +const restrictedUid = require('../utils'); +const { MIGRATION_DATA_CONFIG } = require('../constants/index'); + +const idToString = (id) => { + if (id === null || id === undefined) return ''; + if (typeof id === 'string') return id; + if (typeof id === 'number' || typeof id === 'bigint' || typeof id === 'boolean') return String(id); + if (Array.isArray(id)) return idToString(id[0]); + + if (typeof id === 'object') { + const candidate = id?.id ?? id?.guid ?? id?.value ?? id?.$id ?? id?._id; + if (typeof candidate === 'string' || typeof candidate === 'number' || typeof candidate === 'bigint') { + return String(candidate); + } + + if (typeof id?.toString === 'function' && id?.toString !== Object?.prototype?.toString) { + const str = id.toString(); + if (typeof str === 'string' && str && str !== '[object Object]') return str; + } + + try { + return JSON.stringify(id); + } catch { + return ''; + } + } + + return ''; +}; + +const idCorrector = (id) => { + if (id === null || id === undefined) return id; + + const raw = idToString(id).trim(); + if (!raw) return id; + + const isGuidLike = + /^\{?[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}?$/.test(raw); + + if (isGuidLike) { + return raw.replace(/[-{}]/g, '').toLowerCase(); + } + + return raw.toLowerCase(); +}; + +const uidCorrector = ({ uid } ) => { + if (!uid || typeof uid !== 'string') { + return ''; + } + + let newUid = uid; + + // Note: UIDs starting with numbers and restricted keywords are handled externally in Sitecore + // The prefix is applied in contentTypeMaker function when needed + + // Clean up the UID + newUid = newUid + .replace(/[ -]/g, '_') // Replace spaces and hyphens with underscores + .replace(/[^a-zA-Z0-9_]+/g, '_') // Replace non-alphanumeric characters (except underscore) + .replace(/([A-Z])/g, (match) => `_${match.toLowerCase()}`) // Handle camelCase + .toLowerCase() // Convert to lowercase + .replace(/_+/g, '_') // Replace multiple underscores with single + .replace(/^_|_$/g, ''); // Remove leading/trailing underscores + + // Ensure UID doesn't start with underscore (Contentstack requirement) + if (newUid.startsWith('_')) { + newUid = newUid.substring(1); + } + + return newUid; +}; + +const extractEntries = async (newPath) => { + try { + + const srcFunc = 'extractEntries'; + + const folderName = path.join(newPath,'master', 'sitecore', 'content'); + + const entriesData = []; + if (fs.existsSync(folderName)) { + + const entryPath = read?.(folderName); + for await (const file of entryPath) { + if (file?.endsWith('data.json')) { + const data = await fs.promises.readFile( + path.join(folderName, file), + 'utf8' + ); + const jsonData = JSON.parse(data); + + const { language, template, tid } = jsonData?.item?.$ ?? {}; + const id = idCorrector(jsonData?.item?.$?.id ); + const entries = {}; + entries[id] = { + meta: jsonData?.item?.$, + fields: jsonData?.item?.fields, + }; + const templateIndex = entriesData?.findIndex( + (ele) => ele?.template === template + ); + if (templateIndex >= 0) { + const entry = entriesData?.[templateIndex]?.locale?.[language]; + if (entry !== undefined) { + entry[id] = { + meta: jsonData?.item?.$, + fields: jsonData?.item?.fields, + }; + } else { + entriesData[templateIndex].locale[language] = entries; + } + } else { + const locale = {}; + locale[language] = entries; + entriesData?.push({ template, locale, tid }); + } + } + } + } + const contentTypeUids = fs.readFileSync( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, MIGRATION_DATA_CONFIG?.DATA_MAPPER_DIR, 'contentTypeKey.json'), + 'utf8' + ); + const contentTypes = JSON.parse(contentTypeUids); + if(!fs.existsSync(path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, MIGRATION_DATA_CONFIG?.ENTRIES_DIR_NAME))){ + fs.mkdirSync(path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, MIGRATION_DATA_CONFIG?.ENTRIES_DIR_NAME), { recursive: true }); + } + + + Object.entries(contentTypes).map(([key, value]) => { + console.info(`🚀 ~ extractEntries ~ Processing Content Type UID:`, value, key); + let contentTypeTitle = ''; + const entryPresent = entriesData?.find( + (item) => + item?.tid === key + ); + const AllentryArray = Array.isArray(entryPresent) ? entryPresent : [entryPresent]; + const entriesArray = []; + + if (AllentryArray && AllentryArray?.length > 0) { + //console.info(`🚀 ~ extractEntries ~ AllentryArray:`, AllentryArray); + for(const entry of AllentryArray){ + const locales = entry?.locale && Object?.keys(entry?.locale); + + if(locales?.length <= 0) continue; + + if(!locales) continue; + + for (const locale of locales) { + Object.entries(entry?.locale?.[locale] || {}).map(([uid, item])=>{ + contentTypeTitle = entry?.template; + const otherCmsEntryUid = idCorrector(item?.meta?.id || ''); + entriesArray.push({ + contentTypeUid: key, + entryName: item?.meta?.name, + language: item?.meta?.language, + otherCmsEntryUid: otherCmsEntryUid, + otherCmsCTName: item?.template, + isUpdate: false, + }); + + }) + } + }; + const message = `${srcFunc} Transforming entries of Content Type ${contentTypeTitle} has begun.`; + console.info(message); + + } + const contentType ={ + "id": key, + "status": 1, + "otherCmsTitle": contentTypeTitle, + "otherCmsUid": value, + "isUpdated": false, + "updateAt": "", + "contentstackTitle": contentTypeTitle, + "contentstackUid": value, + "entryMapping": entriesArray, + + } + + if(entriesArray?.length > 0){ + if(fs.existsSync(path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, MIGRATION_DATA_CONFIG?.CONTENT_TYPES_DIR_NAME, `${contentType?.contentstackUid}.json`))){ + const data = fs.readFileSync( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, MIGRATION_DATA_CONFIG?.CONTENT_TYPES_DIR_NAME, `${contentType?.contentstackUid}.json`), + 'utf8' + ); + const existingContentType = JSON.parse(data); + existingContentType.entryMapping = entriesArray; + fs.writeFileSync( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, MIGRATION_DATA_CONFIG?.CONTENT_TYPES_DIR_NAME, `${contentType?.contentstackUid}.json`), + JSON.stringify(existingContentType, null, 4)); + } + + } + + }); + + return true; + } catch (err) { + console.error('🚀 ~ createEntry ~ err:', err); + } +}; + +module.exports = extractEntries; \ No newline at end of file diff --git a/upload-api/migration-wordpress/libs/extractEntries.ts b/upload-api/migration-wordpress/libs/extractEntries.ts new file mode 100644 index 000000000..783cea92f --- /dev/null +++ b/upload-api/migration-wordpress/libs/extractEntries.ts @@ -0,0 +1,190 @@ +import fs from 'fs'; +import path from 'path'; +import config from '../config/index.json'; + +const { contentTypes: contentTypesConfig } = config?.modules; +const contentTypeFolderPath = path.resolve(config?.data, contentTypesConfig?.dirName); + +const EXCLUDED_POST_TYPES = new Set(['attachment', 'wp_global_styles', 'wp_navigation']); + +const normalizeArray = (value: T | T[] | undefined): T[] => { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +}; + +const idCorrector = (id: string) => { + const normalized = id?.replace(/[-{}]/g, ''); + return normalized ? normalized.toLowerCase() : id; +}; + +const getEntryName = (item: any): string => { + if (typeof item?.title === 'string' && item.title.trim()) { + return item.title.trim(); + } + if (item?.title?.text) { + return String(item.title.text).trim(); + } + + // Handle author entries + if (item?.['wp:author_display_name']) { + return String(item['wp:author_display_name']).trim(); + } + if (item?.['wp:author_login']) { + return String(item['wp:author_login']).trim(); + } + + // Handle terms entries + if (item?.['wp:term_name']) { + return String(item['wp:term_name']).trim(); + } + if (item?.['wp:term_slug']) { + return String(item['wp:term_slug']).trim(); + } + + if (typeof item?.['wp:post_name'] === 'string' && item['wp:post_name'].trim()) { + return item['wp:post_name'].trim(); + } + return 'Untitled Entry'; +}; + +/** + * All WordPress source entry keys use `posts_${...}` (any content type) so they align with + * wordpress.service export JSON and CLI uid-mapping. + */ +const getSourceEntryUid = (item: any): string => { + const postId = item?.['wp:post_id']; + if (postId != null && String(postId).trim() !== '') { + return idCorrector(`posts_${postId}`); + } + + const authorId = item?.['wp:author_id']; + if (authorId != null && String(authorId).trim() !== '') { + return idCorrector(`posts_${authorId}`); + } + + const termId = item?.['wp:term_id']; + if (termId != null && String(termId).trim() !== '') { + return idCorrector(`posts_${termId}`); + } + + const candidate = + item?.guid?.text ?? item?.guid ?? item?.link ?? getEntryName(item); + const base = idCorrector(String(candidate || 'entry')); + return idCorrector(`posts_${base}`); +}; + +const getEntryLanguage = (item: any, channelLanguage?: string): string => { + const postMeta = normalizeArray(item?.['wp:postmeta']); + const languageMeta = postMeta.find((meta: any) => { + const key = String(meta?.['wp:meta_key'] || '').toLowerCase(); + return key === 'language' || key === '_language' || key === 'locale' || key === '_locale'; + }); + + const metaLanguage = languageMeta?.['wp:meta_value']; + if (typeof metaLanguage === 'string' && metaLanguage.trim()) { + return metaLanguage.trim(); + } + + if (typeof channelLanguage === 'string' && channelLanguage.trim()) { + return channelLanguage.trim(); + } + + return 'en-us'; +}; + +const extractEntries = async (filePath: string, contentTypeData: any[] = []) => { + try { + const rawData = await fs.promises.readFile(filePath, 'utf8'); + const jsonData = JSON.parse(rawData); + const items = normalizeArray(jsonData?.rss?.channel?.item); + const channelLanguage = jsonData?.rss?.channel?.language; + + const groupedByType = items?.reduce((acc: Record, item: any) => { + const postType = item?.['wp:post_type'] || 'unknown'; + if (EXCLUDED_POST_TYPES.has(postType)) return acc; + if (!acc[postType]) acc[postType] = []; + acc[postType].push(item); + return acc; + }, {}); + + // Extract author entries + const authorData = jsonData?.rss?.channel?.['wp:author']; + if (authorData) { + const authorEntries = normalizeArray(authorData).map((author: any) => ({ + 'wp:post_type': 'author', + 'wp:post_id': author?.['wp:author_id'], + title: author?.['wp:author_display_name'] || author?.['wp:author_login'], + 'wp:author_login': author?.['wp:author_login'], + 'wp:author_email': author?.['wp:author_email'], + 'wp:author_display_name': author?.['wp:author_display_name'], + 'wp:author_first_name': author?.['wp:author_first_name'], + 'wp:author_last_name': author?.['wp:author_last_name'] + })); + if (authorEntries.length > 0) { + groupedByType['author'] = authorEntries; + } + } + + // Extract terms entries (wp:term) + const termData = jsonData?.rss?.channel?.['wp:term']; + if (termData) { + const termEntries = normalizeArray(termData).map((term: any) => ({ + 'wp:post_type': 'terms', + 'wp:post_id': term?.['wp:term_id'], + title: term?.['wp:term_name'] || term?.['wp:term_slug'], + 'wp:term_id': term?.['wp:term_id'], + 'wp:term_taxonomy': term?.['wp:term_taxonomy'], + 'wp:term_slug': term?.['wp:term_slug'], + 'wp:term_name': term?.['wp:term_name'] + })); + if (termEntries.length > 0) { + groupedByType['terms'] = termEntries; + } + } + + const updatedTypes = contentTypeData?.map((ct) => ({ ...ct })); + + for (const [type, entries] of Object.entries(groupedByType)) { + const entryMapping = normalizeArray(entries) + .map((item: any) => { + const otherCmsEntryUid = getSourceEntryUid(item); + if (!otherCmsEntryUid) return null; + return { + contentTypeUid: type, + entryName: getEntryName(item), + otherCmsEntryUid, + otherCmsCTName: type, + language: getEntryLanguage(item, channelLanguage), + isUpdate: false + }; + }) + .filter(Boolean); + + const contentTypeFilePath = path.join(contentTypeFolderPath, `${type.toLowerCase()}.json`); + if (fs.existsSync(contentTypeFilePath)) { + const ctFile = JSON.parse(await fs.promises.readFile(contentTypeFilePath, 'utf8')); + ctFile.entryMapping = entryMapping; + await fs.promises.writeFile(contentTypeFilePath, JSON.stringify(ctFile, null, 4), 'utf8'); + } + + const index = updatedTypes.findIndex( + (ct: any) => + ct?.otherCmsUid?.toLowerCase?.() === type.toLowerCase() || + ct?.contentstackUid?.toLowerCase?.() === type.toLowerCase() + ); + if (index >= 0) { + updatedTypes[index] = { + ...updatedTypes[index], + entryMapping + }; + } + } + + return updatedTypes; + } catch (error: any) { + console.error('Error while extracting WordPress entries:', error?.message || error); + return contentTypeData; + } +}; + +export default extractEntries; diff --git a/upload-api/package.json b/upload-api/package.json index f5fb7f050..acb5f6de2 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -91,4 +91,4 @@ "diff": ">=5.2.2", "qs": ">=6.14.2" } -} +} \ No newline at end of file diff --git a/upload-api/src/config/index.json b/upload-api/src/config/index.json new file mode 100644 index 000000000..21c9104b6 --- /dev/null +++ b/upload-api/src/config/index.json @@ -0,0 +1,29 @@ +{ + "plan": { + "dropdown": { + "optionLimit": 100 + } + }, + "cmsType": "cmsType", + "isLocalPath": true, + "awsData": { + "awsRegion": "us-east-2", + "awsAccessKeyId": "", + "awsSecretAccessKey": "", + "awsSessionToken": "", + "bucketName": "", + "bucketKey": "" + }, + "mysql": { + "host": "host_name", + "user": "user_name", + "password": "", + "database": "database_name", + "port": "port_number" + }, + "assetsConfig": { + "base_url": "drupal_assets_base_url", + "public_path": "drupal_assets_public_path" + }, + "localPath": "your_local_cms_data_path" +} \ No newline at end of file diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts deleted file mode 100644 index 4bde1a7e6..000000000 --- a/upload-api/src/config/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -export default { - plan: { - dropdown: { optionLimit: 100 } - }, - cmsType: process.env.CMS_TYPE || 'cmsType', - isLocalPath: true, - awsData: { - awsRegion: 'us-east-2', - awsAccessKeyId: '', - awsSecretAccessKey: '', - awsSessionToken: '', - bucketName: '', - bucketKey: '' - }, - mysql: { - host: process.env.MYSQL_HOST || 'host_name', - user: process.env.MYSQL_USER || 'user_name', - password: process.env.MYSQL_PASSWORD || '', - database: process.env.MYSQL_DATABASE || 'database_name', - port: process.env.MYSQL_PORT || 'port_number' - }, - assetsConfig: { - base_url: process.env.DRUPAL_ASSETS_BASE_URL || 'drupal_assets_base_url', - public_path: process.env.DRUPAL_ASSETS_PUBLIC_PATH || 'drupal_assets_public_path' - }, - localPath: process.env.CMS_LOCAL_PATH || process.env.CONTAINER_PATH || 'your_local_cms_data_path', -}; diff --git a/upload-api/src/controllers/sitecore/index.ts b/upload-api/src/controllers/sitecore/index.ts index df7db6b4e..8e824dcef 100644 --- a/upload-api/src/controllers/sitecore/index.ts +++ b/upload-api/src/controllers/sitecore/index.ts @@ -12,7 +12,8 @@ const { ExtractConfiguration, reference, ExtractFiles, - extractLocales + extractLocales, + extractEntries } = require('migration-sitecore'); const { CONTENT_TYPES_DIR_NAME, GLOBAL_FIELDS_DIR_NAME, GLOBAL_FIELDS_FILE_NAME } = @@ -141,6 +142,7 @@ const createSitecoreMapper = async ( await createLocaleSource?.({ app_token, localeData, projectId }); await ExtractConfiguration(newPath); await contentTypes(newPath, affix, config); + await extractEntries(newPath); const infoMap = await reference(); if (infoMap?.contentTypeUids?.length) { const fieldMapping: any = { contentTypes: [], extractPath: filePath }; diff --git a/upload-api/src/helper/index.ts b/upload-api/src/helper/index.ts index 07cc83a37..5642d31de 100644 --- a/upload-api/src/helper/index.ts +++ b/upload-api/src/helper/index.ts @@ -209,4 +209,31 @@ function deleteFolderSync(folderPath: string): void { } } -export { getFileName, saveZip, saveJson, fileOperationLimiter, deleteFolderSync, parseXmlToJson }; +async function updateConfigFile(filePath?: string): Promise { + try { + const configFilePath = path.join(process.cwd(), 'src', 'config', 'index.json'); + const config: any = JSON.parse(await fs.promises.readFile(configFilePath, 'utf8')); + + // If filePath is provided and not empty, update the config file + if (filePath && typeof filePath === 'string' && filePath.trim() !== '') { + const resolvedFilePath = path.resolve(filePath.trim()); + + const updatedConfig = { + ...config, + localPath: resolvedFilePath + }; + + const configContent = JSON.stringify(updatedConfig, null, 2); + await fs.promises.writeFile(configFilePath, configContent, 'utf8'); + + return updatedConfig; + } + + return config; + } catch (error) { + logger.error('Error updating config file', { err: error }); + return undefined; + } +} + +export { getFileName, saveZip, saveJson, fileOperationLimiter, deleteFolderSync, parseXmlToJson, updateConfigFile }; diff --git a/upload-api/src/routes/index.ts b/upload-api/src/routes/index.ts index 385dc4d0f..40c0ed384 100644 --- a/upload-api/src/routes/index.ts +++ b/upload-api/src/routes/index.ts @@ -10,11 +10,11 @@ import { UploadPartCommand } from '@aws-sdk/client-s3'; import { client } from '../services/aws/client'; -import { fileOperationLimiter } from '../helper'; +import { fileOperationLimiter, updateConfigFile } from '../helper'; import handleFileProcessing from '../services/fileProcessing'; -import config from '../config/index'; import createMapper from '../services/createMapper'; import { sanitizeId, sanitizeFilename, isPathWithinBase } from '../utils/sanitize-path.utils'; +import logger from '../utils/logger'; const router: Router = express.Router(); // Use memory storage to avoid saving the file locally @@ -98,9 +98,17 @@ router.get( const projectId: string = sanitizeId(req?.headers?.projectid ?? ''); const app_token: string | string[] = req?.headers?.app_token ?? ''; const affix: string = sanitizeId(req?.headers?.affix ?? 'csm'); - const cmsType = config?.cmsType?.toLowerCase(); + const config = await updateConfigFile(); + if (!config) { + logger.error('Failed to load application config'); + return res.status(500).json({ + status: 500, + message: 'Failed to load application configuration' + }); + } + const cmsType = config.cmsType?.toLowerCase(); - if (config?.isLocalPath) { + if (config.isLocalPath) { const localPath = config?.localPath || ''; // Check if localPath indicates a SQL/MySQL connection (case-insensitive) @@ -416,8 +424,15 @@ router.get( ); router.get('/config', async function (req: Request, res: Response) { - // Strip mysql password before sending config to the client - const { password, ...safeMysql } = config?.mysql || {}; + const config = await updateConfigFile(); + if (!config) { + logger.error('Failed to load application config'); + return res.status(500).json({ + status: 500, + message: 'Failed to load application configuration' + }); + } + const { password, ...safeMysql } = config.mysql || {}; const safeConfig = { ...config, mysql: safeMysql diff --git a/upload-api/src/services/aws/client.ts b/upload-api/src/services/aws/client.ts index 1b2c90e5d..2df06353f 100644 --- a/upload-api/src/services/aws/client.ts +++ b/upload-api/src/services/aws/client.ts @@ -1,5 +1,5 @@ import { S3Client } from '@aws-sdk/client-s3'; -import config from '../../config'; +import config from '../../config/index.json'; interface AWSCredentials { accessKeyId: string; @@ -15,6 +15,7 @@ interface S3ClientConfig { //process.env.AWS_ACCESS_KEY_ID ?? //process.env.AWS_SECRET_ACCESS_KEY ?? //process.env.AWS_SESSION_TOKEN ?? + const clientConfig: S3ClientConfig = { region: config?.awsData?.awsRegion, credentials: { diff --git a/upload-api/src/services/fileProcessing.ts b/upload-api/src/services/fileProcessing.ts index de81727e8..8c8e58d28 100644 --- a/upload-api/src/services/fileProcessing.ts +++ b/upload-api/src/services/fileProcessing.ts @@ -1,8 +1,7 @@ import { HTTP_TEXTS, HTTP_CODES } from '../constants'; -import { parseXmlToJson, saveJson, saveZip } from '../helper'; +import { parseXmlToJson, saveJson, saveZip, updateConfigFile } from '../helper'; import JSZip from 'jszip'; import validator from '../validators'; -import config from '../config/index'; import logger from '../utils/logger.js'; const handleFileProcessing = async ( @@ -11,6 +10,15 @@ const handleFileProcessing = async ( cmsType: string, name: string ) => { + const config = await updateConfigFile(); + if (!config) { + logger.error('Failed to load application config'); + return { + status: HTTP_CODES.SERVER_ERROR, + message: HTTP_TEXTS.INTERNAL_ERROR, + file_details: undefined + }; + } if (fileExt === 'zip') { const zip = new JSZip(); await zip.loadAsync(zipBuffer); diff --git a/upload-api/tests/unit/config/index.config.test.ts b/upload-api/tests/unit/config/index.config.test.ts index 73f05093f..7b42d3503 100644 --- a/upload-api/tests/unit/config/index.config.test.ts +++ b/upload-api/tests/unit/config/index.config.test.ts @@ -6,7 +6,7 @@ describe('config/index', () => { }); it('should export default configuration object', async () => { - const config = (await import('../../../src/config/index')).default; + const config = (await import('../../../src/config/index.json')).default; expect(config).toHaveProperty('plan'); expect(config).toHaveProperty('cmsType'); @@ -18,41 +18,17 @@ describe('config/index', () => { }); it('should have plan with dropdown optionLimit', async () => { - const config = (await import('../../../src/config/index')).default; + const config = (await import('../../../src/config/index.json')).default; expect(config.plan.dropdown.optionLimit).toBe(100); }); it('should have isLocalPath as true', async () => { - const config = (await import('../../../src/config/index')).default; + const config = (await import('../../../src/config/index.json')).default; expect(config.isLocalPath).toBe(true); }); - it('should use CMS_TYPE env var when set', async () => { - vi.stubEnv('CMS_TYPE', 'sitecore'); - const config = (await import('../../../src/config/index')).default; - expect(config.cmsType).toBe('sitecore'); - }); - - it('should use CONTAINER_PATH env var when set', async () => { - vi.stubEnv('CONTAINER_PATH', '/custom/path'); - const config = (await import('../../../src/config/index')).default; - expect(config.localPath).toBe('/custom/path'); - }); - - it('should use DRUPAL_ASSETS_BASE_URL env var when set', async () => { - vi.stubEnv('DRUPAL_ASSETS_BASE_URL', 'https://example.com'); - const config = (await import('../../../src/config/index')).default; - expect(config.assetsConfig.base_url).toBe('https://example.com'); - }); - - it('should use DRUPAL_ASSETS_PUBLIC_PATH env var when set', async () => { - vi.stubEnv('DRUPAL_ASSETS_PUBLIC_PATH', '/custom/files'); - const config = (await import('../../../src/config/index')).default; - expect(config.assetsConfig.public_path).toBe('/custom/files'); - }); - it('should have default AWS data', async () => { - const config = (await import('../../../src/config/index')).default; + const config = (await import('../../../src/config/index.json')).default; expect(config.awsData).toEqual({ awsRegion: 'us-east-2', awsAccessKeyId: '', @@ -64,7 +40,7 @@ describe('config/index', () => { }); it('should have default MySQL configuration', async () => { - const config = (await import('../../../src/config/index')).default; + const config = (await import('../../../src/config/index.json')).default; expect(config.mysql.host).toBe('host_name'); expect(config.mysql.user).toBe('user_name'); expect(config.mysql.database).toBe('database_name'); diff --git a/upload-api/tests/unit/helper/index.test.ts b/upload-api/tests/unit/helper/index.test.ts index 66232c178..ec5016885 100644 --- a/upload-api/tests/unit/helper/index.test.ts +++ b/upload-api/tests/unit/helper/index.test.ts @@ -2,10 +2,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; const { mockMkdir, mockWriteFile, mockExistsSync, mockReaddirSync, - mockLstatSync, mockUnlinkSync, mockRmdirSync, mockParseStringPromise, + mockLstatSync, mockUnlinkSync, mockRmdirSync, mockParseStringPromise, mockReadFile, } = vi.hoisted(() => ({ mockMkdir: vi.fn().mockResolvedValue(undefined), mockWriteFile: vi.fn().mockResolvedValue(undefined), + mockReadFile: vi.fn(), mockExistsSync: vi.fn(), mockReaddirSync: vi.fn(), mockLstatSync: vi.fn(), @@ -28,6 +29,7 @@ vi.mock('fs', () => ({ promises: { mkdir: (...a: any[]) => mockMkdir(...a), writeFile: (...a: any[]) => mockWriteFile(...a), + readFile: (...a: any[]) => mockReadFile(...a), }, }, existsSync: (...a: any[]) => mockExistsSync(...a), @@ -38,6 +40,7 @@ vi.mock('fs', () => ({ promises: { mkdir: (...a: any[]) => mockMkdir(...a), writeFile: (...a: any[]) => mockWriteFile(...a), + readFile: (...a: any[]) => mockReadFile(...a), }, })); @@ -55,12 +58,21 @@ vi.mock('jszip', () => { return { default: Cls, __esModule: true }; }); -import { getFileName, saveJson, parseXmlToJson, deleteFolderSync, saveZip } from '../../../src/helper/index'; +import logger from '../../../src/utils/logger'; +import { + getFileName, + saveJson, + parseXmlToJson, + deleteFolderSync, + saveZip, + updateConfigFile, +} from '../../../src/helper/index'; describe('helper/index', () => { beforeEach(() => { vi.clearAllMocks(); mockParseStringPromise.mockResolvedValue({ rss: { channel: { item: [] } } }); + mockReadFile.mockResolvedValue(JSON.stringify({ localPath: '/tmp/old', mode: 'test' })); }); describe('getFileName', () => { @@ -246,4 +258,33 @@ describe('helper/index', () => { expect(result.isSaved).toBe(true); }); }); + + describe('updateConfigFile', () => { + it('returns existing config when filePath is empty', async () => { + const result = await updateConfigFile(''); + expect(result).toEqual({ localPath: '/tmp/old', mode: 'test' }); + expect(mockWriteFile).not.toHaveBeenCalled(); + }); + + it('updates localPath and writes config when filePath is provided', async () => { + const result = await updateConfigFile('/tmp/new-path'); + expect(result).toBeDefined(); + expect(result.localPath).toBe('/tmp/new-path'); + expect(mockWriteFile).toHaveBeenCalledWith( + expect.stringContaining('src/config/index.json'), + expect.stringContaining('"localPath": "/tmp/new-path"'), + 'utf8' + ); + }); + + it('returns undefined when config read fails', async () => { + mockReadFile.mockRejectedValueOnce(new Error('read fail')); + const result = await updateConfigFile('/tmp/new-path'); + expect(result).toBeUndefined(); + expect(logger.error).toHaveBeenCalledWith( + 'Error updating config file', + expect.objectContaining({ err: expect.any(Error) }) + ); + }); + }); }); diff --git a/upload-api/tests/unit/routes/index.routes.test.ts b/upload-api/tests/unit/routes/index.routes.test.ts index 0d82a1072..a6da27796 100644 --- a/upload-api/tests/unit/routes/index.routes.test.ts +++ b/upload-api/tests/unit/routes/index.routes.test.ts @@ -45,6 +45,7 @@ vi.mock('../../../src/services/aws/client', () => ({ vi.mock('../../../src/helper', () => ({ fileOperationLimiter: (_req: any, _res: any, next: any) => next(), deleteFolderSync: vi.fn(), + updateConfigFile: vi.fn().mockImplementation(() => Promise.resolve(mockConfig)), })); vi.mock('../../../src/services/fileProcessing', () => ({ @@ -55,7 +56,7 @@ vi.mock('../../../src/services/createMapper', () => ({ default: (...args: any[]) => mockCreateMapper(...args), })); -vi.mock('../../../src/config/index', () => ({ default: mockConfig })); +vi.mock('../../../src/config/index.json', () => ({ default: mockConfig })); vi.mock('@aws-sdk/client-s3', () => ({ GetObjectCommand: vi.fn().mockImplementation(function (this: any, p: any) { Object.assign(this, p); }), diff --git a/upload-api/tests/unit/services/aws-client.test.ts b/upload-api/tests/unit/services/aws-client.test.ts index 3bab12d4a..d41d880f4 100644 --- a/upload-api/tests/unit/services/aws-client.test.ts +++ b/upload-api/tests/unit/services/aws-client.test.ts @@ -7,7 +7,7 @@ const { mockS3Client } = vi.hoisted(() => ({ }), })); -vi.mock('../../../src/config', () => ({ +vi.mock('../../../src/config/index.json', () => ({ default: { awsData: { awsRegion: 'us-east-2', diff --git a/upload-api/tests/unit/services/fileProcessing.test.ts b/upload-api/tests/unit/services/fileProcessing.test.ts index 2f5be5d54..2513a590b 100644 --- a/upload-api/tests/unit/services/fileProcessing.test.ts +++ b/upload-api/tests/unit/services/fileProcessing.test.ts @@ -1,11 +1,13 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const { mockValidator, mockSaveZip, mockSaveJson, mockParseXmlToJson } = vi.hoisted(() => ({ - mockValidator: vi.fn(), - mockSaveZip: vi.fn(), - mockSaveJson: vi.fn(), - mockParseXmlToJson: vi.fn(), -})); +const { mockValidator, mockSaveZip, mockSaveJson, mockParseXmlToJson, mockUpdateConfigFile } = + vi.hoisted(() => ({ + mockValidator: vi.fn(), + mockSaveZip: vi.fn(), + mockSaveJson: vi.fn(), + mockParseXmlToJson: vi.fn(), + mockUpdateConfigFile: vi.fn(), + })); vi.mock('../../../src/validators/index', () => ({ default: mockValidator })); vi.mock('../../../src/helper/index', () => ({ @@ -15,13 +17,14 @@ vi.mock('../../../src/helper/index', () => ({ fileOperationLimiter: vi.fn(), deleteFolderSync: vi.fn(), getFileName: vi.fn(), + updateConfigFile: (...args: unknown[]) => mockUpdateConfigFile(...args), })); vi.mock('../../../src/utils/logger', () => ({ default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() }, })); -vi.mock('../../../src/config/index', () => ({ +vi.mock('../../../src/config/index.json', () => ({ default: { cmsType: 'wordpress', mysql: { host: 'localhost', user: 'root', password: 'pw', database: 'db', port: '3306' }, @@ -40,6 +43,13 @@ import handleFileProcessing from '../../../src/services/fileProcessing'; describe('handleFileProcessing', () => { beforeEach(() => { vi.clearAllMocks(); + mockUpdateConfigFile.mockResolvedValue({ + cmsType: 'wordpress', + mysql: { host: 'localhost', user: 'root', password: 'pw', database: 'db', port: '3306' }, + assetsConfig: { base_url: 'http://test.com', public_path: '/files' }, + isLocalPath: true, + localPath: '', + }); }); describe('zip files', () => {