diff --git a/.changeset/sync-env-vars-is-secret.md b/.changeset/sync-env-vars-is-secret.md new file mode 100644 index 00000000000..58b38aca64a --- /dev/null +++ b/.changeset/sync-env-vars-is-secret.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/build": patch +--- + +You can now mark environment variables synced via the `syncEnvVars` build extension as secrets. Return `{ name, value, isSecret: true }` from your callback and those variables are stored redacted in the dashboard, just like manually created secret env vars. diff --git a/docs/config/extensions/syncEnvVars.mdx b/docs/config/extensions/syncEnvVars.mdx index 039c5a5ec22..f1014981adb 100644 --- a/docs/config/extensions/syncEnvVars.mdx +++ b/docs/config/extensions/syncEnvVars.mdx @@ -32,6 +32,33 @@ The callback is passed a context object with the following properties: - `projectRef`: The project ref of the Trigger.dev project - `env`: The environment variables that are currently set in the Trigger.dev project +### Marking variables as secrets + +Return the array form and set `isSecret: true` on any variable you want stored as a [secret](/deploy-environment-variables). Secret env vars are redacted in the dashboard and their values can't be revealed after they're set, just like manually created secret variables. You can mix secret and non-secret variables in the same callback. + +```ts +import { defineConfig } from "@trigger.dev/sdk"; +import { syncEnvVars } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + build: { + extensions: [ + syncEnvVars(async (ctx) => { + return [ + { name: "PUBLIC_API_URL", value: "https://api.example.com" }, + { name: "DATABASE_URL", value: "postgres://...", isSecret: true }, + ]; + }), + ], + }, +}); +``` + + + `isSecret` is only available when you return the array form (`{ name, value, isSecret }`). The + record form (`{ KEY: "value" }`) always creates non-secret variables. + + ### Example: Sync env vars from Infisical In this example we're using env vars from [Infisical](https://infisical.com). diff --git a/packages/build/package.json b/packages/build/package.json index 2fd0366c3a5..1588e01fa61 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -74,7 +74,8 @@ "dev": "tshy --watch", "typecheck": "tsc --noEmit -p tsconfig.src.json", "update-version": "tsx ../../scripts/updateVersion.ts", - "check-exports": "attw --pack ." + "check-exports": "attw --pack .", + "test": "vitest" }, "dependencies": { "@prisma/config": "^6.10.0", diff --git a/packages/build/src/extensions/core/syncEnvVars.test.ts b/packages/build/src/extensions/core/syncEnvVars.test.ts new file mode 100644 index 00000000000..d090393e935 --- /dev/null +++ b/packages/build/src/extensions/core/syncEnvVars.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import type { BuildContext, BuildLayer } from "@trigger.dev/core/v3/build"; +import { syncEnvVars, type SyncEnvVarsFunction } from "./syncEnvVars.js"; + +async function runExtension(fn: SyncEnvVarsFunction): Promise { + let captured: BuildLayer | undefined; + + const context = { + target: "deploy", + config: { project: "proj_test" }, + logger: { + spinner: () => ({ stop: () => {} }), + }, + addLayer: (layer: BuildLayer) => { + captured = layer; + }, + } as unknown as BuildContext; + + const manifest = { + deploy: { env: {} }, + environment: "prod", + branch: undefined, + } as any; + + const extension = syncEnvVars(fn); + await extension.onBuildComplete!(context, manifest); + + return captured; +} + +describe("syncEnvVars isSecret", () => { + it("partitions secret and non-secret vars across child and parent", async () => { + const layer = await runExtension(() => [ + { name: "PUBLIC_URL", value: "https://example.com" }, + { name: "API_KEY", value: "secret-key", isSecret: true }, + { name: "PARENT_PUBLIC", value: "parent", isParentEnv: true }, + { name: "PARENT_SECRET", value: "parent-secret", isParentEnv: true, isSecret: true }, + ]); + + expect(layer?.deploy?.env).toEqual({ PUBLIC_URL: "https://example.com" }); + expect(layer?.deploy?.secretEnv).toEqual({ API_KEY: "secret-key" }); + expect(layer?.deploy?.parentEnv).toEqual({ PARENT_PUBLIC: "parent" }); + expect(layer?.deploy?.secretParentEnv).toEqual({ PARENT_SECRET: "parent-secret" }); + }); + + it("treats the record form as all non-secret", async () => { + const layer = await runExtension(() => ({ DATABASE_URL: "postgres://..." })); + + expect(layer?.deploy?.env).toEqual({ DATABASE_URL: "postgres://..." }); + expect(layer?.deploy?.secretEnv).toBeUndefined(); + expect(layer?.deploy?.secretParentEnv).toBeUndefined(); + }); + + it("omits secret buckets when no var is marked secret", async () => { + const layer = await runExtension(() => [{ name: "PUBLIC_URL", value: "https://example.com" }]); + + expect(layer?.deploy?.env).toEqual({ PUBLIC_URL: "https://example.com" }); + expect(layer?.deploy?.secretEnv).toBeUndefined(); + expect(layer?.deploy?.secretParentEnv).toBeUndefined(); + }); +}); diff --git a/packages/build/src/extensions/core/syncEnvVars.ts b/packages/build/src/extensions/core/syncEnvVars.ts index 4b437835615..6da28a05eb0 100644 --- a/packages/build/src/extensions/core/syncEnvVars.ts +++ b/packages/build/src/extensions/core/syncEnvVars.ts @@ -2,7 +2,7 @@ import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; export type SyncEnvVarsBody = | Record - | Array<{ name: string; value: string; isParentEnv?: boolean }>; + | Array<{ name: string; value: string; isParentEnv?: boolean; isSecret?: boolean }>; export type SyncEnvVarsResult = | SyncEnvVarsBody @@ -100,9 +100,16 @@ export function syncEnvVars(fn: SyncEnvVarsFunction, options?: SyncEnvVarsOption const env = stripUnsyncableEnvVars(result.env); const parentEnv = result.parentEnv ? stripUnsyncableEnvVars(result.parentEnv) : undefined; + const secretEnv = result.secretEnv ? stripUnsyncableEnvVars(result.secretEnv) : undefined; + const secretParentEnv = result.secretParentEnv + ? stripUnsyncableEnvVars(result.secretParentEnv) + : undefined; const numberOfEnvVars = - Object.keys(env).length + (parentEnv ? Object.keys(parentEnv).length : 0); + Object.keys(env).length + + (parentEnv ? Object.keys(parentEnv).length : 0) + + (secretEnv ? Object.keys(secretEnv).length : 0) + + (secretParentEnv ? Object.keys(secretParentEnv).length : 0); if (numberOfEnvVars === 0) { $spinner.stop("No env vars detected"); @@ -119,6 +126,8 @@ export function syncEnvVars(fn: SyncEnvVarsFunction, options?: SyncEnvVarsOption deploy: { env, parentEnv, + secretEnv, + secretParentEnv, override: options?.override ?? true, }, }); @@ -151,9 +160,22 @@ async function callSyncEnvVarsFn( environment: string, branch: string | undefined, context: BuildContext -): Promise<{ env: Record; parentEnv?: Record } | undefined> { +): Promise< + | { + env: Record; + parentEnv?: Record; + secretEnv?: Record; + secretParentEnv?: Record; + } + | undefined +> { if (syncEnvVarsFn && typeof syncEnvVarsFn === "function") { - let resolvedEnvVars: { env: Record; parentEnv?: Record } = { + let resolvedEnvVars: { + env: Record; + parentEnv?: Record; + secretEnv?: Record; + secretParentEnv?: Record; + } = { env: {}, }; let result; @@ -184,10 +206,16 @@ async function callSyncEnvVarsFn( typeof item.value === "string" ) { if (item.isParentEnv) { - if (!resolvedEnvVars.parentEnv) { - resolvedEnvVars.parentEnv = {}; + if (item.isSecret) { + resolvedEnvVars.secretParentEnv ??= {}; + resolvedEnvVars.secretParentEnv[item.name] = item.value; + } else { + resolvedEnvVars.parentEnv ??= {}; + resolvedEnvVars.parentEnv[item.name] = item.value; } - resolvedEnvVars.parentEnv[item.name] = item.value; + } else if (item.isSecret) { + resolvedEnvVars.secretEnv ??= {}; + resolvedEnvVars.secretEnv[item.name] = item.value; } else { resolvedEnvVars.env[item.name] = item.value; } diff --git a/packages/build/vitest.config.ts b/packages/build/vitest.config.ts new file mode 100644 index 00000000000..c497b8ec974 --- /dev/null +++ b/packages/build/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + globals: true, + }, +}); diff --git a/packages/cli-v3/src/build/extensions.ts b/packages/cli-v3/src/build/extensions.ts index e38fe903d84..1f545034246 100644 --- a/packages/cli-v3/src/build/extensions.ts +++ b/packages/cli-v3/src/build/extensions.ts @@ -185,6 +185,46 @@ function applyLayerToManifest(layer: BuildLayer, manifest: BuildManifest): Build } } + if (layer.deploy?.secretEnv) { + $manifest.deploy.env ??= {}; + $manifest.deploy.sync ??= {}; + $manifest.deploy.sync.secretEnv ??= {}; + + for (const [key, value] of Object.entries(layer.deploy.secretEnv)) { + if (!value) { + continue; + } + + if (layer.deploy.override || $manifest.deploy.env[key] === undefined) { + const existingValue = $manifest.deploy.env[key]; + + if (existingValue !== value) { + $manifest.deploy.sync.secretEnv[key] = value; + } + } + } + } + + if (layer.deploy?.secretParentEnv) { + $manifest.deploy.env ??= {}; + $manifest.deploy.sync ??= {}; + $manifest.deploy.sync.secretParentEnv ??= {}; + + for (const [key, value] of Object.entries(layer.deploy.secretParentEnv)) { + if (!value) { + continue; + } + + if (layer.deploy.override || $manifest.deploy.env[key] === undefined) { + const existingValue = $manifest.deploy.env[key]; + + if (existingValue !== value) { + $manifest.deploy.sync.secretParentEnv[key] = value; + } + } + } + } + if (layer.dependencies) { const externals = $manifest.externals ?? []; diff --git a/packages/cli-v3/src/commands/deploy.ts b/packages/cli-v3/src/commands/deploy.ts index ad0faa8b31c..0c6eece0a54 100644 --- a/packages/cli-v3/src/commands/deploy.ts +++ b/packages/cli-v3/src/commands/deploy.ts @@ -454,16 +454,23 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { } } + const childVars = buildManifest.deploy.sync?.env ?? {}; + const parentVars = buildManifest.deploy.sync?.parentEnv ?? {}; + const secretChildVars = buildManifest.deploy.sync?.secretEnv ?? {}; + const secretParentVars = buildManifest.deploy.sync?.secretParentEnv ?? {}; + const hasVarsToSync = - Object.keys(buildManifest.deploy.sync?.env || {}).length > 0 || + Object.keys(childVars).length > 0 || + Object.keys(secretChildVars).length > 0 || // Only sync parent variables if this is a branch environment - (branch && Object.keys(buildManifest.deploy.sync?.parentEnv || {}).length > 0); + (branch && (Object.keys(parentVars).length > 0 || Object.keys(secretParentVars).length > 0)); if (hasVarsToSync) { - const childVars = buildManifest.deploy.sync?.env ?? {}; - const parentVars = buildManifest.deploy.sync?.parentEnv ?? {}; - - const numberOfEnvVars = Object.keys(childVars).length + Object.keys(parentVars).length; + const numberOfEnvVars = + Object.keys(childVars).length + + Object.keys(parentVars).length + + Object.keys(secretChildVars).length + + Object.keys(secretParentVars).length; const vars = numberOfEnvVars === 1 ? "var" : "vars"; if (!options.skipSyncEnvVars) { @@ -475,7 +482,9 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) { resolvedConfig.project, options.env, childVars, - parentVars + parentVars, + secretChildVars, + secretParentVars ); if (!uploadResult.success) { @@ -768,13 +777,41 @@ export async function syncEnvVarsWithServer( projectRef: string, environmentSlug: string, envVars: Record, - parentEnvVars?: Record + parentEnvVars?: Record, + secretEnvVars?: Record, + secretParentEnvVars?: Record ) { - return await apiClient.importEnvVars(projectRef, environmentSlug, { - variables: envVars, - parentVariables: parentEnvVars, - override: true, - }); + const hasNonSecret = + Object.keys(envVars).length > 0 || Object.keys(parentEnvVars ?? {}).length > 0; + const hasSecret = + Object.keys(secretEnvVars ?? {}).length > 0 || + Object.keys(secretParentEnvVars ?? {}).length > 0; + + // The import API applies isSecret per call, so secret and non-secret vars go in separate calls. + // Default to success so an all-empty call (no vars to sync) is a no-op, not undefined. + let result: Awaited> = { + success: true, + data: { success: true }, + }; + + if (hasNonSecret) { + result = await apiClient.importEnvVars(projectRef, environmentSlug, { + variables: envVars, + parentVariables: parentEnvVars, + override: true, + }); + } + + if (hasSecret && result.success) { + result = await apiClient.importEnvVars(projectRef, environmentSlug, { + variables: secretEnvVars ?? {}, + parentVariables: secretParentEnvVars, + override: true, + isSecret: true, + }); + } + + return result; } async function failDeploy( diff --git a/packages/cli-v3/src/commands/workers/build.ts b/packages/cli-v3/src/commands/workers/build.ts index 9148d7ee51a..cea001fe99b 100644 --- a/packages/cli-v3/src/commands/workers/build.ts +++ b/packages/cli-v3/src/commands/workers/build.ts @@ -259,12 +259,23 @@ async function _workerBuildCommand(dir: string, options: WorkersBuildCommandOpti local = true; } - if ( - buildManifest.deploy.sync && - buildManifest.deploy.sync.env && - Object.keys(buildManifest.deploy.sync.env).length > 0 - ) { - const numberOfEnvVars = Object.keys(buildManifest.deploy.sync.env).length; + const childVars = buildManifest.deploy.sync?.env ?? {}; + const parentVars = buildManifest.deploy.sync?.parentEnv ?? {}; + const secretChildVars = buildManifest.deploy.sync?.secretEnv ?? {}; + const secretParentVars = buildManifest.deploy.sync?.secretParentEnv ?? {}; + + const hasVarsToSync = + Object.keys(childVars).length > 0 || + Object.keys(secretChildVars).length > 0 || + // Only sync parent variables if this is a branch environment + (branch && (Object.keys(parentVars).length > 0 || Object.keys(secretParentVars).length > 0)); + + if (hasVarsToSync) { + const numberOfEnvVars = + Object.keys(childVars).length + + Object.keys(parentVars).length + + Object.keys(secretChildVars).length + + Object.keys(secretParentVars).length; const vars = numberOfEnvVars === 1 ? "var" : "vars"; if (!options.skipSyncEnvVars) { @@ -274,8 +285,10 @@ async function _workerBuildCommand(dir: string, options: WorkersBuildCommandOpti projectClient.client, resolvedConfig.project, options.env, - buildManifest.deploy.sync.env, - buildManifest.deploy.sync.parentEnv + childVars, + parentVars, + secretChildVars, + secretParentVars ); if (!success) { @@ -451,15 +464,39 @@ export async function syncEnvVarsWithServer( projectRef: string, environmentSlug: string, envVars: Record, - parentEnvVars?: Record + parentEnvVars?: Record, + secretEnvVars?: Record, + secretParentEnvVars?: Record ) { - const uploadResult = await apiClient.importEnvVars(projectRef, environmentSlug, { - variables: envVars, - parentVariables: parentEnvVars, - override: true, - }); + const hasNonSecret = + Object.keys(envVars).length > 0 || Object.keys(parentEnvVars ?? {}).length > 0; + const hasSecret = + Object.keys(secretEnvVars ?? {}).length > 0 || + Object.keys(secretParentEnvVars ?? {}).length > 0; + + // The import API applies isSecret per call, so secret and non-secret vars go in separate calls. + let success = true; + + if (hasNonSecret) { + const result = await apiClient.importEnvVars(projectRef, environmentSlug, { + variables: envVars, + parentVariables: parentEnvVars, + override: true, + }); + success = result.success; + } + + if (hasSecret && success) { + const result = await apiClient.importEnvVars(projectRef, environmentSlug, { + variables: secretEnvVars ?? {}, + parentVariables: secretParentEnvVars, + override: true, + isSecret: true, + }); + success = result.success; + } - return uploadResult.success; + return success; } async function failDeploy( diff --git a/packages/core/src/v3/build/extensions.ts b/packages/core/src/v3/build/extensions.ts index 3b809040eaa..6b461985567 100644 --- a/packages/core/src/v3/build/extensions.ts +++ b/packages/core/src/v3/build/extensions.ts @@ -63,6 +63,8 @@ export interface BuildLayer { deploy?: { env?: Record; parentEnv?: Record; + secretEnv?: Record; + secretParentEnv?: Record; override?: boolean; }; dependencies?: Record; diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index 52a353aaf6d..eeeaba73116 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -53,6 +53,8 @@ export const BuildManifest = z.object({ .object({ env: z.record(z.string()).optional(), parentEnv: z.record(z.string()).optional(), + secretEnv: z.record(z.string()).optional(), + secretParentEnv: z.record(z.string()).optional(), }) .optional(), }),