Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
"transfer": "tsx src/cli.ts",
"ts-check": "yarn tsc --noEmit",
"test": "vitest run",
"test:coverage": "vitest run --coverage --coverage.include=\"src/**/**\"",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage --coverage.include=\"src/**/**\"",
"format": "oxfmt",
"format:fix": "oxfmt",
"format:check": "oxfmt --check",
Expand Down
2 changes: 2 additions & 0 deletions projects/v5-to-v6/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ SOURCE_REGION=us-east-1
# SOURCE_AWS_SESSION_TOKEN=
SOURCE_DDB_TABLE=webiny-v5-table
SOURCE_S3_BUCKET=webiny-v5-files
SOURCE_ACCOUNT_ID=
SOURCE_AUDIT_LOGS_TABLE=
SOURCE_OS_TABLE=webiny-v5-es-table

Expand All @@ -28,6 +29,7 @@ TARGET_REGION=us-east-1
# TARGET_AWS_SESSION_TOKEN=
TARGET_DDB_TABLE=webiny-v6-table
TARGET_S3_BUCKET=webiny-v6-files
TARGET_ACCOUNT_ID=
TARGET_AUDIT_LOGS_TABLE=webiny-v6-audit-logs
TARGET_OS_TABLE=webiny-v6-es-table
TARGET_OS_ENDPOINT=https://search-my-domain.us-east-1.es.amazonaws.com
Expand Down
21 changes: 6 additions & 15 deletions src/commands/run/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { fileURLToPath } from "node:url";
import { join } from "node:path";
import { readdir, readFile } from "node:fs/promises";
import { execa } from "execa";
import { confirm } from "@inquirer/prompts";
import { bootstrap } from "~/bootstrap.ts";
import { formatError } from "~/base/index.ts";
import type { RunStats } from "~/features/PipelineRunner/abstractions/PipelineRunner.ts";
Expand Down Expand Up @@ -110,8 +109,14 @@ export async function handler(
logger.info(` ok ${entry.label}`);
} else if (entry.status === "denied") {
logger.error(` DENIED ${entry.label}`);
if (entry.hint) {
logger.error(` ${entry.hint}`);
}
} else if (entry.status === "missing") {
logger.error(` MISSING ${entry.label}`);
if (entry.hint) {
logger.error(` ${entry.hint}`);
}
} else {
logger.warn(` unknown ${entry.label}`);
}
Expand All @@ -123,20 +128,6 @@ export async function handler(
throw new AccessCheckError(blocked.length);
}

const guardWarnings = (
await Promise.all(runner.getProcessors().map(p => p.getGuardWarning?.() ?? null))
).filter((w): w is string => w !== null);

if (guardWarnings.length > 0) {
for (const warning of guardWarnings) {
logger.warn(warning);
}
const proceed = await confirm({ message: "Proceed with transfer?" });
if (!proceed) {
process.exit(0);
}
}

const beforeHook = container.resolve(BeforeTransferHook);
logger.info("Running before-transfer hooks...");
await beforeHook.execute();
Expand Down
9 changes: 1 addition & 8 deletions src/domain/pipeline/abstractions/Processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,6 @@ interface IProcessor<
*/
checkAccess(): Promise<AccessCheck.Entry[]>;

/**
* Pre-transfer guard check. Called in the orchestrator before any segment
* workers are spawned. Return a human-readable warning string when the
* processor detects a condition that requires user confirmation (e.g.
* cross-account S3 copy), or null to proceed silently.
*/
getGuardWarning?(): Promise<string | null>;

/**
* Drain the processor's commands from the bag and write to target. The
* act of calling commands.get(key) marks that key as "claimed" — the
Expand All @@ -84,6 +76,7 @@ export namespace AccessCheck {
export interface Entry {
label: string;
status: Status;
hint?: string;
}

export type Report = Entry[];
Expand Down
4 changes: 4 additions & 0 deletions src/domain/transform/filters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,7 @@ export const isFormBuilderRecord = (record: BaseRecord): boolean => {
}
return type.startsWith("fb.form.") || type.startsWith("fb.formSubmission");
};

export const isAdminUser = (record: BaseRecord): boolean => {
return record.PK.includes("#SECURITY#USER#") && record.GSI1_PK === "securityRole#full-access";
};
56 changes: 38 additions & 18 deletions src/features/S3Processor/S3Processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,13 @@ class S3ProcessorImpl implements Processor.Interface<
// No onEnd — S3 has no sensible per-record default. Transformers call
// ctx.copyFile(...) explicitly when they want to emit a copy.

public async getGuardWarning(): Promise<string | null> {
public async checkAccess(): Promise<AccessCheck.Entry[]> {
const sourceAccount = this.config.source.accountId || null;
const targetAccount = this.config.target.accountId || null;
if (sourceAccount === null || targetAccount === null || sourceAccount === targetAccount) {
return null;
}
return (
`S3 file copy is cross-account: source account ${sourceAccount} → target account ${targetAccount}.\n` +
`CopyObject runs with target credentials — the source bucket "${this.config.source.s3.bucket}"\n` +
`must have a bucket policy granting account ${targetAccount} s3:GetObject access.`
);
}
const isCrossAccount =
sourceAccount !== null && targetAccount !== null && sourceAccount !== targetAccount;

public async checkAccess(): Promise<AccessCheck.Entry[]> {
const [sourceEntry, targetEntry] = await Promise.all([
const checks: Promise<AccessCheck.Entry>[] = [
this.headBucket(
this.config.source.credentials,
this.config.source.region,
Expand All @@ -70,29 +62,57 @@ class S3ProcessorImpl implements Processor.Interface<
this.config.target.s3.bucket,
"target"
)
]);
return [sourceEntry, targetEntry];
];

if (isCrossAccount) {
checks.push(
this.headBucketWithLabel(
this.config.target.credentials,
this.config.source.region,
this.config.source.s3.bucket,
`S3 cross-account read (target credentials → source bucket: ${this.config.source.s3.bucket})`,
`S3 CopyObject runs with target credentials. Add a bucket policy on ` +
`"${this.config.source.s3.bucket}" granting s3:GetObject to account ${targetAccount}.`
)
);
}

return Promise.all(checks);
}

private async headBucket(
private headBucket(
credentials: MigrationConfig.Interface["source"]["credentials"],
region: string,
bucket: string,
side: string
): Promise<AccessCheck.Entry> {
const label = `S3 ${side} bucket: ${bucket}`;
return this.headBucketWithLabel(
credentials,
region,
bucket,
`S3 ${side} bucket: ${bucket}`
);
}

private async headBucketWithLabel(
credentials: MigrationConfig.Interface["source"]["credentials"],
region: string,
bucket: string,
label: string,
hint?: string
): Promise<AccessCheck.Entry> {
const client = new S3({ region, credentials: credentials as never });
try {
await client.headBucket({ Bucket: bucket });
return { label, status: "ok" };
} catch (error) {
if (isAccessDeniedError(error)) {
return { label, status: "denied" };
return { label, status: "denied", hint };
}
const errName = (error as AwsErrorLike).name ?? (error as AwsErrorLike).code;
const httpStatus = (error as AwsErrorLike).$metadata?.httpStatusCode;
if (errName === "NoSuchBucket" || httpStatus === 404) {
return { label, status: "missing" };
return { label, status: "missing", hint };
}
return { label, status: "unknown" };
} finally {
Expand Down
11 changes: 11 additions & 0 deletions src/presets/v5-to-v6-ddb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { createFilter } from "~/domain/pipeline/Filter.ts";
import {
byType,
isAcoSearchRecord,
isAdminUser,
isAuditLogEntry,
isBackgroundTask,
isBuiltInSecurityRole,
Expand Down Expand Up @@ -287,6 +288,15 @@ export default createTransferPreset({
.blackhole()
.build();

const adminUsers = factory
.create({
name: "AdminUsers",
scanner: DdbScanner,
processors: [DdbProcessor]
})
.filter(createFilter(isAdminUser))
.build();

// ========================================================================
// Register pipelines with runner
// IMPORTANT: Order matters due to first-match-wins behavior
Expand All @@ -305,6 +315,7 @@ export default createTransferPreset({
.register(cmsModels)
.register(folderPermissions)
.register(cmsEntries)
.register(adminUsers)
.register(formBuilderRecords);
}
});
2 changes: 2 additions & 0 deletions templates/internal-project/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ SOURCE_REGION={{SOURCE_REGION}}

SOURCE_DDB_TABLE={{SOURCE_DDB_TABLE}}
SOURCE_S3_BUCKET={{SOURCE_S3_BUCKET}}
SOURCE_ACCOUNT_ID={{SOURCE_ACCOUNT_ID}}
SOURCE_AUDIT_LOGS_TABLE={{SOURCE_AUDIT_LOGS_TABLE}}
SOURCE_OS_TABLE={{SOURCE_OS_TABLE}}

Expand All @@ -34,6 +35,7 @@ TARGET_REGION={{TARGET_REGION}}

TARGET_DDB_TABLE={{TARGET_DDB_TABLE}}
TARGET_S3_BUCKET={{TARGET_S3_BUCKET}}
TARGET_ACCOUNT_ID={{TARGET_ACCOUNT_ID}}
TARGET_AUDIT_LOGS_TABLE={{TARGET_AUDIT_LOGS_TABLE}}
TARGET_OS_TABLE={{TARGET_OS_TABLE}}
TARGET_OS_ENDPOINT={{TARGET_OS_ENDPOINT}}
Expand Down
2 changes: 2 additions & 0 deletions templates/projects/example/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ SOURCE_REGION={{SOURCE_REGION}}

SOURCE_DDB_TABLE={{SOURCE_DDB_TABLE}}
SOURCE_S3_BUCKET={{SOURCE_S3_BUCKET}}
SOURCE_ACCOUNT_ID={{SOURCE_ACCOUNT_ID}}
SOURCE_OS_TABLE={{SOURCE_OS_TABLE}}

# --- Target environment ------------------------------------------------
Expand All @@ -35,6 +36,7 @@ TARGET_REGION={{TARGET_REGION}}

TARGET_DDB_TABLE={{TARGET_DDB_TABLE}}
TARGET_S3_BUCKET={{TARGET_S3_BUCKET}}
TARGET_ACCOUNT_ID={{TARGET_ACCOUNT_ID}}
TARGET_OS_TABLE={{TARGET_OS_TABLE}}
TARGET_OS_ENDPOINT={{TARGET_OS_ENDPOINT}}
TARGET_OS_INDEX_PREFIX={{TARGET_OS_INDEX_PREFIX}}
Expand Down
Loading