Orphaned Assets Fix

client/modules/IDE/actions/project.js

@@ -306,6 +306,8 @@ export function cloneProject(project) {
     generateNewIdsForChildren(rootFile, newFiles);
 
     // duplicate all files hosted on S3
+    const copiedS3Assets = [];
+
     each(
       newFiles,
       (file, callback) => {
@@ -316,24 +318,20 @@ export function cloneProject(project) {
           (file.url.includes(S3_BUCKET_URL_BASE) ||
             file.url.includes(S3_BUCKET))
         ) {
-          const formParams = {
-            url: file.url
-          };
-          apiClient.post('/S3/copy', formParams).then((response) => {
+          apiClient.post('/S3/copy', { url: file.url }).then((response) => {
             file.url = response.data.url;
+            copiedS3Assets.push(response.data.url);
             callback(null);
           });
         } else {
           callback(null);
         }
       },
-      (err) => {
-        // if not errors in duplicating the files on S3, then duplicate it
-        const formParams = Object.assign(
-          {},
-          { name: `${projectName} copy` },
-          { files: newFiles }
-        );
+      () => {
+        const formParams = {
+          name: `${projectName} copy`,
+          files: newFiles
+        };
         apiClient
           .post('/projects', formParams)
           .then((response) => {
@@ -343,6 +341,12 @@ export function cloneProject(project) {
             dispatch(setNewProject(response.data));
           })
           .catch((error) => {
+            copiedS3Assets.forEach((url) => {
+              const objectKey = url.split('/').pop();
+              apiClient
+                .delete(`/S3/delete?objectKey=${objectKey}`)
+                .catch(() => {});
+            });
             dispatch({
               type: ActionTypes.PROJECT_SAVE_FAIL,
               error: error?.response?.data

server/controllers/aws.controller.js

@@ -156,7 +156,7 @@ export async function signS3(req, res) {
     const acl = 'public-read';
     const policy = S3Policy.generate({
       acl,
-      key: `${req.body.userId}/${filename}`,
+      key: `pending/${req.user.id}/${filename}`,
       bucket: process.env.S3_BUCKET,
       contentType: req.body.type,
       region: process.env.AWS_REGION,

server/controllers/project.controller.js

@@ -11,6 +11,7 @@ import Project from '../models/project';
 import { User } from '../models/user';
 import { resolvePathToFile } from '../utils/filePath';
 import { generateFileSystemSafeName } from '../utils/generateFileSystemSafeName';
+import { commitPendingAssets } from '../utils/pendingAssets';
 
 const s3Client = new S3Client({
   credentials: {
@@ -77,6 +78,13 @@ export async function updateProject(req, res) {
     )
       .populate('user', 'username')
       .exec();
+
+    try {
+      await commitPendingAssets(req.user.id);
+    } catch (error) {
+      console.error('Error committing pending assets:', error);
+    }
+
     if (
       req.body.files &&
       updatedProject.files.length !== req.body.files.length

server/controllers/project.controller/createProject.js

@@ -4,6 +4,7 @@ import {
   FileValidationError,
   ProjectValidationError
 } from '../../domain-objects/Project';
+import { commitPendingAssets } from '../../utils/pendingAssets';
 
 export default function createProject(req, res) {
   const projectValues = Object.assign({}, req.body, { user: req.user._id });
@@ -16,7 +17,12 @@ export default function createProject(req, res) {
     return Project.populate(newProject, {
       path: 'user',
       select: 'username'
-    }).then((newProjectWithUser) => {
+    }).then(async (newProjectWithUser) => {
+      try {
+        await commitPendingAssets(req.user.id);
+      } catch (error) {
+        console.error('Error committing pending assets:', error);
+      }
       res.json(newProjectWithUser);
     });
   }
@@ -87,6 +93,13 @@ export async function apiCreateProject(req, res) {
     }
 
     const newProject = await model.save();
+
+    try {
+      await commitPendingAssets(req.user.id);
+    } catch (error) {
+      console.error('Error committing pending assets:', error);
+    }
+
     res.status(201).json({ id: newProject.id });
   } catch (err) {
     handleErrors(err);

server/server.js

@@ -27,6 +27,7 @@ import serverRoutes from './routes/server.routes';
 import redirectEmbedRoutes from './routes/redirectEmbed.routes';
 import passportRoutes from './routes/passport.routes';
 import { requestsOfTypeJSON } from './utils/requestsOfType';
+import { cleanupStalePendingAssets } from './utils/pendingAssets';
 
 import { renderIndex } from './views/index';
 import { get404Sketch } from './views/404Page';
@@ -219,4 +220,8 @@ app.listen(process.env.PORT, (error) => {
   }
 });
 
+// Set up periodic cleanup of stale pending assets
+const cleanupIntervalMs = 24 * 60 * 60 * 1000;
+setInterval(cleanupStalePendingAssets, cleanupIntervalMs);
+
 export default app;

server/utils/pendingAssets.js

@@ -0,0 +1,113 @@
+import {
+  S3Client,
+  CopyObjectCommand,
+  ListObjectsCommand,
+  DeleteObjectsCommand
+} from '@aws-sdk/client-s3';
+
+const s3Client = new S3Client({
+  credentials: {
+    accessKeyId: process.env.AWS_ACCESS_KEY,
+    secretAccessKey: process.env.AWS_SECRET_KEY
+  },
+  region: process.env.AWS_REGION
+});
+
+const STALE_ASSET_TIME = 7 * 24 * 60;
+
+async function getPendingAssets(userId) {
+  const params = {
+    Bucket: process.env.S3_BUCKET,
+    Prefix: `pending/${userId}/`
+  };
+
+  try {
+    const data = await s3Client.send(new ListObjectsCommand(params));
+    return data.Contents || [];
+  } catch (error) {
+    console.error('Error listing pending assets from S3:', error);
+    throw error;
+  }
+}
+
+async function getStalePendingAssets(minutesOld = STALE_ASSET_TIME) {
+  const params = {
+    Bucket: process.env.S3_BUCKET,
+    Prefix: 'pending/'
+  };
+
+  try {
+    const data = await s3Client.send(new ListObjectsCommand(params));
+    if (!data.Contents) return [];
+
+    const cutoffTime = new Date();
+    cutoffTime.setMinutes(cutoffTime.getMinutes() - minutesOld);
+
+    return data.Contents.filter(
+      (object) => object.LastModified < cutoffTime
+    ).map((object) => object.Key);
+  } catch (error) {
+    console.error('Error listing stale pending assets from S3:', error);
+    throw error;
+  }
+}
+
+async function moveAssetFromPending(pendingKey, userId) {
+  const filename = pendingKey.split('/').pop();
+  const destinationKey = `${userId}/${filename}`;
+
+  await s3Client.send(
+    new CopyObjectCommand({
+      Bucket: process.env.S3_BUCKET,
+      CopySource: `${process.env.S3_BUCKET}/${pendingKey}`,
+      Key: destinationKey,
+      ACL: 'public-read'
+    })
+  );
+
+  await s3Client.send(
+    new DeleteObjectsCommand({
+      Bucket: process.env.S3_BUCKET,
+      Delete: { Objects: [{ Key: pendingKey }] }
+    })
+  );
+
+  return destinationKey;
+}
+
+async function deleteKeys(keys) {
+  if (keys.length === 0) return;
+
+  await s3Client.send(
+    new DeleteObjectsCommand({
+      Bucket: process.env.S3_BUCKET,
+      Delete: { Objects: keys.map((key) => ({ Key: key })) }
+    })
+  );
+}
+
+export async function commitPendingAssets(userId) {
+  try {
+    const assets = await getPendingAssets(userId);
+    if (assets.length === 0) return [];
+
+    const movePromises = assets.map((asset) =>
+      moveAssetFromPending(asset.Key, userId)
+    );
+    return Promise.all(movePromises);
+  } catch (error) {
+    console.error('Error committing pending assets:', error);
+    throw error;
+  }
+}
+
+export async function cleanupStalePendingAssets() {
+  try {
+    const staleKeys = await getStalePendingAssets();
+    if (staleKeys.length > 0) {
+      await deleteKeys(staleKeys);
+    }
+  } catch (error) {
+    console.error('Error cleaning up stale pending assets:', error);
+  }
+}