diff --git a/client/modules/IDE/actions/project.js b/client/modules/IDE/actions/project.js index 7787f879bb..9577c8e427 100644 --- a/client/modules/IDE/actions/project.js +++ b/client/modules/IDE/actions/project.js @@ -306,6 +306,8 @@ export function cloneProject(project) { generateNewIdsForChildren(rootFile, newFiles); // duplicate all files hosted on S3 + const copiedS3Assets = []; + each( newFiles, (file, callback) => { @@ -316,24 +318,20 @@ export function cloneProject(project) { (file.url.includes(S3_BUCKET_URL_BASE) || file.url.includes(S3_BUCKET)) ) { - const formParams = { - url: file.url - }; - apiClient.post('/S3/copy', formParams).then((response) => { + apiClient.post('/S3/copy', { url: file.url }).then((response) => { file.url = response.data.url; + copiedS3Assets.push(response.data.url); callback(null); }); } else { callback(null); } }, - (err) => { - // if not errors in duplicating the files on S3, then duplicate it - const formParams = Object.assign( - {}, - { name: `${projectName} copy` }, - { files: newFiles } - ); + () => { + const formParams = { + name: `${projectName} copy`, + files: newFiles + }; apiClient .post('/projects', formParams) .then((response) => { @@ -343,6 +341,12 @@ export function cloneProject(project) { dispatch(setNewProject(response.data)); }) .catch((error) => { + copiedS3Assets.forEach((url) => { + const objectKey = url.split('/').pop(); + apiClient + .delete(`/S3/delete?objectKey=${objectKey}`) + .catch(() => {}); + }); dispatch({ type: ActionTypes.PROJECT_SAVE_FAIL, error: error?.response?.data diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index b6e03db13c..88807b2aa6 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -156,7 +156,7 @@ export async function signS3(req, res) { const acl = 'public-read'; const policy = S3Policy.generate({ acl, - key: `${req.body.userId}/${filename}`, + key: `pending/${req.user.id}/${filename}`, bucket: process.env.S3_BUCKET, contentType: req.body.type, region: process.env.AWS_REGION, diff --git a/server/controllers/project.controller.js b/server/controllers/project.controller.js index 57eda81381..887629d2f8 100644 --- a/server/controllers/project.controller.js +++ b/server/controllers/project.controller.js @@ -11,6 +11,7 @@ import Project from '../models/project'; import { User } from '../models/user'; import { resolvePathToFile } from '../utils/filePath'; import { generateFileSystemSafeName } from '../utils/generateFileSystemSafeName'; +import { commitPendingAssets } from '../utils/pendingAssets'; const s3Client = new S3Client({ credentials: { @@ -77,6 +78,13 @@ export async function updateProject(req, res) { ) .populate('user', 'username') .exec(); + + try { + await commitPendingAssets(req.user.id); + } catch (error) { + console.error('Error committing pending assets:', error); + } + if ( req.body.files && updatedProject.files.length !== req.body.files.length diff --git a/server/controllers/project.controller/createProject.js b/server/controllers/project.controller/createProject.js index 001e007155..5d5f51eff2 100644 --- a/server/controllers/project.controller/createProject.js +++ b/server/controllers/project.controller/createProject.js @@ -4,6 +4,7 @@ import { FileValidationError, ProjectValidationError } from '../../domain-objects/Project'; +import { commitPendingAssets } from '../../utils/pendingAssets'; export default function createProject(req, res) { const projectValues = Object.assign({}, req.body, { user: req.user._id }); @@ -16,7 +17,12 @@ export default function createProject(req, res) { return Project.populate(newProject, { path: 'user', select: 'username' - }).then((newProjectWithUser) => { + }).then(async (newProjectWithUser) => { + try { + await commitPendingAssets(req.user.id); + } catch (error) { + console.error('Error committing pending assets:', error); + } res.json(newProjectWithUser); }); } @@ -87,6 +93,13 @@ export async function apiCreateProject(req, res) { } const newProject = await model.save(); + + try { + await commitPendingAssets(req.user.id); + } catch (error) { + console.error('Error committing pending assets:', error); + } + res.status(201).json({ id: newProject.id }); } catch (err) { handleErrors(err); diff --git a/server/server.js b/server/server.js index cfaa318646..7905fe01c2 100644 --- a/server/server.js +++ b/server/server.js @@ -27,6 +27,7 @@ import serverRoutes from './routes/server.routes'; import redirectEmbedRoutes from './routes/redirectEmbed.routes'; import passportRoutes from './routes/passport.routes'; import { requestsOfTypeJSON } from './utils/requestsOfType'; +import { cleanupStalePendingAssets } from './utils/pendingAssets'; import { renderIndex } from './views/index'; import { get404Sketch } from './views/404Page'; @@ -219,4 +220,8 @@ app.listen(process.env.PORT, (error) => { } }); +// Set up periodic cleanup of stale pending assets +const cleanupIntervalMs = 24 * 60 * 60 * 1000; +setInterval(cleanupStalePendingAssets, cleanupIntervalMs); + export default app; diff --git a/server/utils/pendingAssets.js b/server/utils/pendingAssets.js new file mode 100644 index 0000000000..c8df4217c0 --- /dev/null +++ b/server/utils/pendingAssets.js @@ -0,0 +1,113 @@ +import { + S3Client, + CopyObjectCommand, + ListObjectsCommand, + DeleteObjectsCommand +} from '@aws-sdk/client-s3'; + +const s3Client = new S3Client({ + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY, + secretAccessKey: process.env.AWS_SECRET_KEY + }, + region: process.env.AWS_REGION +}); + +const STALE_ASSET_TIME = 7 * 24 * 60; + +async function getPendingAssets(userId) { + const params = { + Bucket: process.env.S3_BUCKET, + Prefix: `pending/${userId}/` + }; + + try { + const data = await s3Client.send(new ListObjectsCommand(params)); + return data.Contents || []; + } catch (error) { + console.error('Error listing pending assets from S3:', error); + throw error; + } +} + +async function getStalePendingAssets(minutesOld = STALE_ASSET_TIME) { + const params = { + Bucket: process.env.S3_BUCKET, + Prefix: 'pending/' + }; + + try { + const data = await s3Client.send(new ListObjectsCommand(params)); + if (!data.Contents) return []; + + const cutoffTime = new Date(); + cutoffTime.setMinutes(cutoffTime.getMinutes() - minutesOld); + + return data.Contents.filter( + (object) => object.LastModified < cutoffTime + ).map((object) => object.Key); + } catch (error) { + console.error('Error listing stale pending assets from S3:', error); + throw error; + } +} + +async function moveAssetFromPending(pendingKey, userId) { + const filename = pendingKey.split('/').pop(); + const destinationKey = `${userId}/${filename}`; + + await s3Client.send( + new CopyObjectCommand({ + Bucket: process.env.S3_BUCKET, + CopySource: `${process.env.S3_BUCKET}/${pendingKey}`, + Key: destinationKey, + ACL: 'public-read' + }) + ); + + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: process.env.S3_BUCKET, + Delete: { Objects: [{ Key: pendingKey }] } + }) + ); + + return destinationKey; +} + +async function deleteKeys(keys) { + if (keys.length === 0) return; + + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: process.env.S3_BUCKET, + Delete: { Objects: keys.map((key) => ({ Key: key })) } + }) + ); +} + +export async function commitPendingAssets(userId) { + try { + const assets = await getPendingAssets(userId); + if (assets.length === 0) return []; + + const movePromises = assets.map((asset) => + moveAssetFromPending(asset.Key, userId) + ); + return Promise.all(movePromises); + } catch (error) { + console.error('Error committing pending assets:', error); + throw error; + } +} + +export async function cleanupStalePendingAssets() { + try { + const staleKeys = await getStalePendingAssets(); + if (staleKeys.length > 0) { + await deleteKeys(staleKeys); + } + } catch (error) { + console.error('Error cleaning up stale pending assets:', error); + } +}