From 91b77673766f5270eb43b717f27eaa65cbd2a0ad Mon Sep 17 00:00:00 2001 From: yugalkaushik Date: Sun, 17 May 2026 17:09:08 +0530 Subject: [PATCH 1/4] Orphaned assets fix --- client/modules/IDE/actions/project.js | 30 +++-- server/controllers/aws.controller.js | 2 +- server/controllers/project.controller.js | 8 ++ .../project.controller/createProject.js | 15 ++- server/server.js | 5 + server/utils/pendingAssets.js | 113 ++++++++++++++++++ 6 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 server/utils/pendingAssets.js diff --git a/client/modules/IDE/actions/project.js b/client/modules/IDE/actions/project.js index 7787f879bb..c645a52252 100644 --- a/client/modules/IDE/actions/project.js +++ b/client/modules/IDE/actions/project.js @@ -298,14 +298,14 @@ export function cloneProject(project) { const projectName = project ? project.name : state.project.name; const newFiles = files.map((file) => ({ ...file })); - // generate new IDS for all files const rootFile = newFiles.find((file) => file.name === 'root'); const newRootFileId = objectID().toHexString(); rootFile.id = newRootFileId; rootFile._id = newRootFileId; generateNewIdsForChildren(rootFile, newFiles); - // duplicate all files hosted on S3 + const copiedS3Assets = []; + each( newFiles, (file, callback) => { @@ -316,24 +316,20 @@ export function cloneProject(project) { (file.url.includes(S3_BUCKET_URL_BASE) || file.url.includes(S3_BUCKET)) ) { - const formParams = { - url: file.url - }; - apiClient.post('/S3/copy', formParams).then((response) => { + apiClient.post('/S3/copy', { url: file.url }).then((response) => { file.url = response.data.url; + copiedS3Assets.push(response.data.url); callback(null); }); } else { callback(null); } }, - (err) => { - // if not errors in duplicating the files on S3, then duplicate it - const formParams = Object.assign( - {}, - { name: `${projectName} copy` }, - { files: newFiles } - ); + () => { + const formParams = { + name: `${projectName} copy`, + files: newFiles + }; apiClient .post('/projects', formParams) .then((response) => { @@ -343,6 +339,14 @@ export function cloneProject(project) { dispatch(setNewProject(response.data)); }) .catch((error) => { + copiedS3Assets.forEach((url) => { + const objectKey = url.split('/').pop(); + apiClient + .delete(`/S3/delete?objectKey=${objectKey}`) + .catch(() => { + // Silently ignore cleanup errors + }); + }); dispatch({ type: ActionTypes.PROJECT_SAVE_FAIL, error: error?.response?.data diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index b6e03db13c..88807b2aa6 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -156,7 +156,7 @@ export async function signS3(req, res) { const acl = 'public-read'; const policy = S3Policy.generate({ acl, - key: `${req.body.userId}/${filename}`, + key: `pending/${req.user.id}/${filename}`, bucket: process.env.S3_BUCKET, contentType: req.body.type, region: process.env.AWS_REGION, diff --git a/server/controllers/project.controller.js b/server/controllers/project.controller.js index 57eda81381..887629d2f8 100644 --- a/server/controllers/project.controller.js +++ b/server/controllers/project.controller.js @@ -11,6 +11,7 @@ import Project from '../models/project'; import { User } from '../models/user'; import { resolvePathToFile } from '../utils/filePath'; import { generateFileSystemSafeName } from '../utils/generateFileSystemSafeName'; +import { commitPendingAssets } from '../utils/pendingAssets'; const s3Client = new S3Client({ credentials: { @@ -77,6 +78,13 @@ export async function updateProject(req, res) { ) .populate('user', 'username') .exec(); + + try { + await commitPendingAssets(req.user.id); + } catch (error) { + console.error('Error committing pending assets:', error); + } + if ( req.body.files && updatedProject.files.length !== req.body.files.length diff --git a/server/controllers/project.controller/createProject.js b/server/controllers/project.controller/createProject.js index 001e007155..5d5f51eff2 100644 --- a/server/controllers/project.controller/createProject.js +++ b/server/controllers/project.controller/createProject.js @@ -4,6 +4,7 @@ import { FileValidationError, ProjectValidationError } from '../../domain-objects/Project'; +import { commitPendingAssets } from '../../utils/pendingAssets'; export default function createProject(req, res) { const projectValues = Object.assign({}, req.body, { user: req.user._id }); @@ -16,7 +17,12 @@ export default function createProject(req, res) { return Project.populate(newProject, { path: 'user', select: 'username' - }).then((newProjectWithUser) => { + }).then(async (newProjectWithUser) => { + try { + await commitPendingAssets(req.user.id); + } catch (error) { + console.error('Error committing pending assets:', error); + } res.json(newProjectWithUser); }); } @@ -87,6 +93,13 @@ export async function apiCreateProject(req, res) { } const newProject = await model.save(); + + try { + await commitPendingAssets(req.user.id); + } catch (error) { + console.error('Error committing pending assets:', error); + } + res.status(201).json({ id: newProject.id }); } catch (err) { handleErrors(err); diff --git a/server/server.js b/server/server.js index 755022c2fd..e1625a0ce5 100644 --- a/server/server.js +++ b/server/server.js @@ -27,6 +27,7 @@ import serverRoutes from './routes/server.routes'; import redirectEmbedRoutes from './routes/redirectEmbed.routes'; import passportRoutes from './routes/passport.routes'; import { requestsOfTypeJSON } from './utils/requestsOfType'; +import { cleanupStalePendingAssets } from './utils/pendingAssets'; import { renderIndex } from './views/index'; import { get404Sketch } from './views/404Page'; @@ -217,4 +218,8 @@ app.listen(process.env.PORT, (error) => { } }); +// Set up periodic cleanup of stale pending assets +const cleanupIntervalMs = 5 * 60 * 1000; +setInterval(cleanupStalePendingAssets, cleanupIntervalMs); + export default app; diff --git a/server/utils/pendingAssets.js b/server/utils/pendingAssets.js new file mode 100644 index 0000000000..438218528f --- /dev/null +++ b/server/utils/pendingAssets.js @@ -0,0 +1,113 @@ +import { + S3Client, + CopyObjectCommand, + ListObjectsCommand, + DeleteObjectsCommand +} from '@aws-sdk/client-s3'; + +const s3Client = new S3Client({ + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY, + secretAccessKey: process.env.AWS_SECRET_KEY + }, + region: process.env.AWS_REGION +}); + +const STALE_ASSET_TIME = 5; + +async function getPendingAssets(userId) { + const params = { + Bucket: process.env.S3_BUCKET, + Prefix: `pending/${userId}/` + }; + + try { + const data = await s3Client.send(new ListObjectsCommand(params)); + return data.Contents || []; + } catch (error) { + console.error('Error listing pending assets from S3:', error); + throw error; + } +} + +async function getStalePendingAssets(minutesOld = STALE_ASSET_TIME) { + const params = { + Bucket: process.env.S3_BUCKET, + Prefix: 'pending/' + }; + + try { + const data = await s3Client.send(new ListObjectsCommand(params)); + if (!data.Contents) return []; + + const cutoffTime = new Date(); + cutoffTime.setMinutes(cutoffTime.getMinutes() - minutesOld); + + return data.Contents.filter( + (object) => object.LastModified < cutoffTime + ).map((object) => object.Key); + } catch (error) { + console.error('Error listing stale pending assets from S3:', error); + throw error; + } +} + +async function moveAssetFromPending(pendingKey, userId) { + const filename = pendingKey.split('/').pop(); + const destinationKey = `${userId}/${filename}`; + + await s3Client.send( + new CopyObjectCommand({ + Bucket: process.env.S3_BUCKET, + CopySource: `${process.env.S3_BUCKET}/${pendingKey}`, + Key: destinationKey, + ACL: 'public-read' + }) + ); + + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: process.env.S3_BUCKET, + Delete: { Objects: [{ Key: pendingKey }] } + }) + ); + + return destinationKey; +} + +async function deleteKeys(keys) { + if (keys.length === 0) return; + + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: process.env.S3_BUCKET, + Delete: { Objects: keys.map((key) => ({ Key: key })) } + }) + ); +} + +export async function commitPendingAssets(userId) { + try { + const assets = await getPendingAssets(userId); + if (assets.length === 0) return []; + + const movePromises = assets.map((asset) => + moveAssetFromPending(asset.Key, userId) + ); + return Promise.all(movePromises); + } catch (error) { + console.error('Error committing pending assets:', error); + throw error; + } +} + +export async function cleanupStalePendingAssets() { + try { + const staleKeys = await getStalePendingAssets(); + if (staleKeys.length > 0) { + await deleteKeys(staleKeys); + } + } catch (error) { + console.error('Error cleaning up stale pending assets:', error); + } +} From 5e6f9da4b36fefd7719a40183a015fc03827e59c Mon Sep 17 00:00:00 2001 From: yugalkaushik Date: Sun, 17 May 2026 17:27:17 +0530 Subject: [PATCH 2/4] changed cleanup threshold to 7 days --- server/server.js | 2 +- server/utils/pendingAssets.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/server.js b/server/server.js index e1625a0ce5..fab88b5e8c 100644 --- a/server/server.js +++ b/server/server.js @@ -219,7 +219,7 @@ app.listen(process.env.PORT, (error) => { }); // Set up periodic cleanup of stale pending assets -const cleanupIntervalMs = 5 * 60 * 1000; +const cleanupIntervalMs = 24 * 60 * 60 * 1000; setInterval(cleanupStalePendingAssets, cleanupIntervalMs); export default app; diff --git a/server/utils/pendingAssets.js b/server/utils/pendingAssets.js index 438218528f..c8df4217c0 100644 --- a/server/utils/pendingAssets.js +++ b/server/utils/pendingAssets.js @@ -13,7 +13,7 @@ const s3Client = new S3Client({ region: process.env.AWS_REGION }); -const STALE_ASSET_TIME = 5; +const STALE_ASSET_TIME = 7 * 24 * 60; async function getPendingAssets(userId) { const params = { From ce83d08e9feb9499565151b7393e60f7ac8d6efd Mon Sep 17 00:00:00 2001 From: yugalkaushik Date: Sun, 17 May 2026 17:40:42 +0530 Subject: [PATCH 3/4] Orphaned Assets Fix --- client/modules/IDE/actions/project.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/modules/IDE/actions/project.js b/client/modules/IDE/actions/project.js index c645a52252..c53f442724 100644 --- a/client/modules/IDE/actions/project.js +++ b/client/modules/IDE/actions/project.js @@ -343,9 +343,7 @@ export function cloneProject(project) { const objectKey = url.split('/').pop(); apiClient .delete(`/S3/delete?objectKey=${objectKey}`) - .catch(() => { - // Silently ignore cleanup errors - }); + .catch(() => {}); }); dispatch({ type: ActionTypes.PROJECT_SAVE_FAIL, From cfb0f2063c5da457395e497b4b1b718ee77c2d15 Mon Sep 17 00:00:00 2001 From: yugalkaushik Date: Sun, 17 May 2026 20:15:14 +0530 Subject: [PATCH 4/4] revert few comments --- client/modules/IDE/actions/project.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/modules/IDE/actions/project.js b/client/modules/IDE/actions/project.js index c53f442724..9577c8e427 100644 --- a/client/modules/IDE/actions/project.js +++ b/client/modules/IDE/actions/project.js @@ -298,12 +298,14 @@ export function cloneProject(project) { const projectName = project ? project.name : state.project.name; const newFiles = files.map((file) => ({ ...file })); + // generate new IDS for all files const rootFile = newFiles.find((file) => file.name === 'root'); const newRootFileId = objectID().toHexString(); rootFile.id = newRootFileId; rootFile._id = newRootFileId; generateNewIdsForChildren(rootFile, newFiles); + // duplicate all files hosted on S3 const copiedS3Assets = []; each(