feat: implement S3 integration for persistent report storage and downloads

Author: Protik111

.github/workflows/aws-ec2-deploy.yml

@@ -64,7 +64,9 @@ jobs:
           cd infra-pulumi
           PUBLIC_IP=$(pulumi stack output publicIp --stack dev)
           echo "public_ip=${PUBLIC_IP}" >> $GITHUB_OUTPUT
-          echo "EC2 Public IP: ${PUBLIC_IP}"
+          
+          BUCKET=$(pulumi stack output reportsBucket --stack dev)
+          echo "reports_bucket=${BUCKET}" >> $GITHUB_OUTPUT
           
           echo "private_key<<EOF" >> $GITHUB_OUTPUT
           pulumi stack output privateKey --stack dev --show-secrets >> $GITHUB_OUTPUT
@@ -82,65 +84,50 @@ jobs:
         uses: appleboy/ssh-action@master
         env:
           DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }}
-          REPORTS_BUCKET_NAME: distributed-job-reports
+          REPORTS_BUCKET_NAME: ${{ steps.ec2.outputs.reports_bucket }}
+          AWS_REGION: ap-southeast-1
         with:
           host: ${{ steps.ec2.outputs.public_ip }}
           username: ec2-user
           key: ${{ steps.ec2.outputs.private_key }}
           timeout: 120s
-          envs: DOCKERHUB_USERNAME,REPORTS_BUCKET_NAME
+          envs: DOCKERHUB_USERNAME,REPORTS_BUCKET_NAME,AWS_REGION
           script: |
             set -e
-            echo "Starting deployment..."
-            
             cd ~/distributed-report-queue
-
-            echo "Syncing code from GitHub..."
             git fetch origin
             git reset --hard origin/main
 
-            echo "Creating .env file..."
             cat > .env << EOF
             REDIS_HOST=redis
             REDIS_PORT=6379
             REDIS_URL=redis://redis:6379
             DOCKERHUB_USERNAME=$DOCKERHUB_USERNAME
             DOCKER_IMAGE_TAG=latest
             NODE_ENV=production
-            REPORTS_BUCKET=$REPORTS_BUCKET_NAME
-            AWS_REGION=ap-southeast-1
+            REPORTS_BUCKET_NAME=$REPORTS_BUCKET_NAME
+            AWS_REGION=$AWS_REGION
             EOF
 
-            echo "Verifying .env content (DOCKERHUB_USERNAME should be visible)..."
-            grep DOCKERHUB_USERNAME .env
-
-            # Copy .env to infra for docker-compose interpolation
             cp .env infra/.env
-
             cd infra
-            echo "Stopping existing services..."
             docker-compose -f docker-compose.prod.yml down --remove-orphans
-            
-            echo "Pulling latest images..."
             docker-compose -f docker-compose.prod.yml pull
-            
-            echo "Starting services..."
             docker-compose -f docker-compose.prod.yml up -d --scale worker=2
-            
-            echo "Cleaning up old images..."
             docker system prune -af --volumes=false
-            echo "Deployment script finished successfully."
 
       - name: Verify Deployment
         uses: appleboy/ssh-action@master
         env:
           DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }}
+          REPORTS_BUCKET_NAME: ${{ steps.ec2.outputs.reports_bucket }}
+          AWS_REGION: ap-southeast-1
         with:
           host: ${{ steps.ec2.outputs.public_ip }}
           username: ec2-user
           key: ${{ steps.ec2.outputs.private_key }}
           timeout: 120s
-          envs: DOCKERHUB_USERNAME
+          envs: DOCKERHUB_USERNAME,REPORTS_BUCKET_NAME,AWS_REGION
           script: |
             set -e
             cd ~/distributed-report-queue/infra

infra-pulumi/index.ts

@@ -26,6 +26,58 @@ const keyPair = new aws.ec2.KeyPair("worker-key-pair", {
     publicKey: sshKey.publicKeyOpenssh,
 }, { provider });
 
+// S3 Bucket for reports
+const bucket = new aws.s3.BucketV2("reports-bucket", {
+    bucket: `distributed-job-reports-${githubUsername}`,
+    forceDestroy: true, // Allow deletion if not empty during cleanup
+}, { provider });
+
+// Enable Public Access Block (disable it to allow public read)
+const bucketPublicAccessBlock = new aws.s3.BucketPublicAccessBlock("reports-bucket-pab", {
+    bucket: bucket.id,
+    blockPublicAcls: false,
+    blockPublicPolicy: false,
+    ignorePublicAcls: false,
+    restrictPublicBuckets: false,
+}, { provider });
+
+// IAM Role for EC2
+const ec2Role = new aws.iam.Role("worker-ec2-role", {
+    assumeRolePolicy: JSON.stringify({
+        Version: "2012-10-17",
+        Statement: [{
+            Action: "sts:AssumeRole",
+            Effect: "Allow",
+            Principal: { Service: "ec2.amazonaws.com" },
+        }],
+    }),
+}, { provider });
+
+// IAM Policy for S3 access
+const s3Policy = new aws.iam.RolePolicy("worker-s3-policy", {
+    role: ec2Role.id,
+    policy: pulumi.all([bucket.arn]).apply(([arn]) => JSON.stringify({
+        Version: "2012-10-17",
+        Statement: [
+            {
+                Action: ["s3:PutObject", "s3:PutObjectAcl", "s3:GetObject"],
+                Effect: "Allow",
+                Resource: [`${arn}/*`],
+            },
+            {
+                Action: ["s3:ListBucket"],
+                Effect: "Allow",
+                Resource: [arn],
+            },
+        ],
+    })),
+}, { provider });
+
+// Instance Profile
+const instanceProfile = new aws.iam.InstanceProfile("worker-instance-profile", {
+    role: ec2Role.name,
+}, { provider });
+
 // VPC
 const vpc = new aws.ec2.Vpc("job-queue-vpc", {
   cidrBlock: "10.0.0.0/16",
@@ -161,6 +213,7 @@ const ec2Instance = new aws.ec2.Instance("worker-instance", {
   vpcSecurityGroupIds: [sgWorkers.id],
   keyName: keyPair.keyName,
   userData: userDataScript,
+  iamInstanceProfile: instanceProfile.name,
   associatePublicIpAddress: true,
   rootBlockDevice: {
     volumeSize: 20,
@@ -174,6 +227,6 @@ const ec2Instance = new aws.ec2.Instance("worker-instance", {
 export const publicIp = ec2Instance.publicIp;
 export const publicDns = ec2Instance.publicDns;
 export const ec2InstanceId = ec2Instance.id;
-export const reportsBucket = "distributed-job-reports";
+export const reportsBucket = bucket.id;
 export const dockerHubUsernameOut = dockerHubUsername;
 export const privateKey = sshKey.privateKeyPem;

infra/docker-compose.prod.yml

@@ -55,6 +55,8 @@ services:
       - REDIS_PORT=6379
       - REDIS_URL=redis://redis:6379
       - NODE_ENV=production
+      - REPORTS_BUCKET_NAME=${REPORTS_BUCKET_NAME}
+      - AWS_REGION=${AWS_REGION:-ap-southeast-1}
     depends_on: [ redis ]
 
   dashboard-frontend:

worker/package.json

@@ -17,7 +17,8 @@
     "express": "^5.2.1",
     "ioredis": "5.3.2",
     "pino": "^8.21.0",
-    "puppeteer": "^24.37.5"
+    "puppeteer": "^24.37.5",
+    "@aws-sdk/client-s3": "^3.540.0"
   },
   "pnpm": {
     "overrides": {

worker/src/lib/s3.service.ts

@@ -0,0 +1,39 @@
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+import logger from "../utils/logger";
+
+const region = process.env.AWS_REGION || "ap-southeast-1";
+const bucketName = process.env.REPORTS_BUCKET_NAME;
+
+// Explicitly not providing credentials to use IAM Instance Profile
+const s3Client = new S3Client({ region });
+
+export const uploadToS3 = async (
+  buffer: Buffer,
+  fileName: string,
+  contentType: string = "application/pdf"
+): Promise<string> => {
+  if (!bucketName) {
+    throw new Error("REPORTS_BUCKET_NAME environment variable is not set");
+  }
+
+  try {
+    const command = new PutObjectCommand({
+      Bucket: bucketName,
+      Key: fileName,
+      Body: buffer,
+      ContentType: contentType,
+      ACL: "public-read", // Make the report publicly downloadable
+    });
+
+    await s3Client.send(command);
+
+    // Construct the public URL
+    const url = `https://${bucketName}.s3.${region}.amazonaws.com/${fileName}`;
+    logger.info({ fileName, bucketName, url }, "File uploaded to S3 successfully");
+    
+    return url;
+  } catch (error: any) {
+    logger.error({ error: error.message, fileName }, "S3 Upload failed");
+    throw error;
+  }
+};

worker/src/processors/report.processor.ts

@@ -6,6 +6,7 @@ import fs from "fs";
 import path from "path";
 import logger from "../utils/logger";
 import redisConnection from "../lib/redis";
+import { uploadToS3 } from "../lib/s3.service";
 
 export interface IReportJobData {
   reportType: string;
@@ -104,10 +105,10 @@ export async function processReportJob(
     await browser.close();
     browser = null;
 
-    // 4. Save Locally
+    // 4. Upload to S3
     await job.updateProgress(80);
     const fileName = `${reportType}-${jobId}-${Date.now()}.pdf`;
-    const reportUrl = await saveToLocalDisk(pdfBuffer, fileName);
+    const reportUrl = await uploadToS3(pdfBuffer, fileName);
 
     // 5. Store Result in Redis (Metadata)
     await job.updateProgress(100);