diff --git a/cdk/src/constructs/orchestration-reconciler.ts b/cdk/src/constructs/orchestration-reconciler.ts new file mode 100644 index 000000000..1b0b85a2c --- /dev/null +++ b/cdk/src/constructs/orchestration-reconciler.ts @@ -0,0 +1,176 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import * as path from 'path'; +import { Duration } from 'aws-cdk-lib'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import { Architecture, FilterCriteria, FilterRule, Runtime, StartingPosition } from 'aws-cdk-lib/aws-lambda'; +import { DynamoEventSource, SqsDlq } from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; +import { TERMINAL_STATUSES } from './task-status'; + +/** + * Properties for OrchestrationReconciler construct. + */ +export interface OrchestrationReconcilerProps { + /** + * TaskTable — MUST have a stream enabled (NEW_IMAGE). This construct is + * the table's stream consumer; the reconciler reacts to child tasks + * reaching terminal status. + */ + readonly taskTable: dynamodb.ITable; + + /** OrchestrationTable — the reconciler reads the DAG + writes child statuses. */ + readonly orchestrationTable: dynamodb.ITable; + + /** Orchestrator function ARN — releaseChild → createTaskCore invokes it. */ + readonly orchestratorFunctionArn?: string; + + /** Forwarded so released child tasks land in the right tables. */ + readonly taskEventsTable: dynamodb.ITable; +} + +/** + * TaskTable-stream consumer that drives Linear parent/sub-issue + * orchestration. On each child task reaching a + * terminal status it releases newly-unblocked children in dependency + * order (see `handlers/orchestration-reconciler.ts`). + * + * Stream-source rationale: TaskEventsTable's stream is at its 2-consumer + * limit (FanOutConsumer + ApprovalMetricsPublisher); TaskTable had no + * stream, so the reconciler is its first and only consumer — zero + * contention with the fan-out plane. + */ + +/** DLQ message retention (days) — long enough for an operator to inspect a + * poison stream record before it ages out. */ +const DLQ_RETENTION_DAYS = 14; + +export class OrchestrationReconciler extends Construct { + public readonly fn: lambda.NodejsFunction; + public readonly dlq: sqs.Queue; + + constructor(scope: Construct, id: string, props: OrchestrationReconcilerProps) { + super(scope, id); + + const handlersDir = path.join(__dirname, '..', 'handlers'); + + this.fn = new lambda.NodejsFunction(this, 'ReconcilerFn', { + entry: path.join(handlersDir, 'orchestration-reconciler.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + timeout: Duration.minutes(2), + // 512 MB (not 256): the reconciler bundles createTaskCore, which + // pulls in the Bedrock guardrail + S3 attachment-screening SDK + // stack. At 256 MB it OOMs during init on every stream event + // (Max Memory Used 255/256 MB) and never releases children. The + // LinearIntegration webhook processor runs the same code at 512 MB. + memorySize: 512, + environment: { + ORCHESTRATION_TABLE_NAME: props.orchestrationTable.tableName, + TASK_TABLE_NAME: props.taskTable.tableName, + TASK_EVENTS_TABLE_NAME: props.taskEventsTable.tableName, + ...(props.orchestratorFunctionArn && { + ORCHESTRATOR_FUNCTION_ARN: props.orchestratorFunctionArn, + }), + }, + bundling: { + externalModules: ['@aws-sdk/*'], + // pdf-parse (v2, pdfjs-based) can't be esbuild-bundled — its pdfjs/native + // deps break at import. The reconciler screens the parent issue's PDF + // attachments, so ship pdf-parse + // unbundled to resolve natively at runtime. MUST match the webhook + // processors' attachment-screening bundling — a Lambda that CALLS + // attachment-screening but omits this carve-out fails every PDF at + // runtime while passing every unit test — a failure mode observed in + // practice on the attachment path. + nodeModules: ['pdf-parse'], + }, + }); + + // DLQ for poison stream records (a record that repeatedly fails the + // reconcile). Fan-out uses the same pattern; without it a bad record + // would block the shard. + this.dlq = new sqs.Queue(this, 'ReconcilerDlq', { + retentionPeriod: Duration.days(DLQ_RETENTION_DAYS), + enforceSSL: true, + }); + + // Orchestration child creation/gating reads + writes the DAG table, + // reads/writes TaskTable (createTaskCore), and writes task events. + props.orchestrationTable.grantReadWriteData(this.fn); + props.taskTable.grantReadWriteData(this.fn); + props.taskEventsTable.grantReadWriteData(this.fn); + + // Subscribe to the TaskTable stream. LATEST: we only care about + // tasks transitioning to terminal from here on. bisectBatchOnError + + // DLQ so one poison record can't wedge the shard. + // + // FilterCriteria: the handler ignores every non-terminal status + // (parseTerminalTaskRecord returns null unless status ∈ TERMINAL), so the + // stream itself filters to terminal statuses. This keeps RUNNING/HYDRATING/ + // heartbeat/progress writes — the bulk of TaskTable churn platform-wide — + // from ever invoking this 512MB reconciler. Behavior-preserving: the records + // dropped here are exactly the ones the handler already discarded. One filter + // pattern per terminal status (FilterCriteria ORs the array). + const terminalFilters = TERMINAL_STATUSES.map((s) => FilterCriteria.filter({ + dynamodb: { NewImage: { status: { S: FilterRule.isEqual(s) } } }, + })); + this.fn.addEventSource(new DynamoEventSource(props.taskTable, { + startingPosition: StartingPosition.LATEST, + batchSize: 10, + retryAttempts: 3, + bisectBatchOnError: true, + onFailure: new SqsDlq(this.dlq), + filters: terminalFilters, + // Partial-batch reporting. Without this, the handler + // returning void meant ANY thrown record failed the WHOLE batch — a single + // poison/throttled record re-drove all its siblings (re-reconciling + // healthy children) until it aged out. With reportBatchItemFailures the + // handler returns only the failed record's sequence number, so just that + // record retries + bisects toward the DLQ while its siblings commit. + reportBatchItemFailures: true, + })); + + NagSuppressions.addResourceSuppressions(this.fn, [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaBasicExecutionRole is required for CloudWatch Logs access', + }, + { + id: 'AwsSolutions-IAM5', + reason: + 'DynamoDB index/* + stream ARN wildcards generated by CDK grantReadWriteData ' + + '(ChildTaskIndex query) and the DynamoEventSource read access', + }, + ], true); + + NagSuppressions.addResourceSuppressions(this.dlq, [ + { + id: 'AwsSolutions-SQS3', + reason: + 'This queue IS the DLQ for the reconciler stream consumer — having its own DLQ would be infinite recursion', + }, + ]); + } +} diff --git a/cdk/src/constructs/stranded-orchestration-reconciler.ts b/cdk/src/constructs/stranded-orchestration-reconciler.ts new file mode 100644 index 000000000..6c87a4722 --- /dev/null +++ b/cdk/src/constructs/stranded-orchestration-reconciler.ts @@ -0,0 +1,126 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import * as path from 'path'; +import { Duration } from 'aws-cdk-lib'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as events from 'aws-cdk-lib/aws-events'; +import * as targets from 'aws-cdk-lib/aws-events-targets'; +import { Architecture, Runtime } from 'aws-cdk-lib/aws-lambda'; +import * as lambda from 'aws-cdk-lib/aws-lambda-nodejs'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; + +/** + * Properties for StrandedOrchestrationReconciler construct. + */ +export interface StrandedOrchestrationReconcilerProps { + /** OrchestrationTable — read DAG state, write recovered child statuses. */ + readonly orchestrationTable: dynamodb.ITable; + /** TaskTable — read released children's task status (terminal? built?) + createTaskCore writes. */ + readonly taskTable: dynamodb.ITable; + /** TaskEventsTable — createTaskCore writes task_created events. */ + readonly taskEventsTable: dynamodb.ITable; + /** Orchestrator function ARN — releaseChild → createTaskCore async-invokes it. */ + readonly orchestratorFunctionArn?: string; + /** + * Sweep cadence. Long enough to amortise the scan; short enough to + * clear a lost-event stall in a reasonable user-facing time. + * @default Duration.minutes(10) + */ + readonly schedule?: Duration; +} + +/** + * Scheduled backstop for sub-issue orchestration. + * + * The live ``OrchestrationReconciler`` reacts to TaskTable-stream terminal + * events to release dependency-unblocked children. If it is unavailable + * when an event fires (deploy/throttle/OOM/DLQ-parked record) that event + * is lost and the orchestration stalls. This scheduled sweep re-derives + * gating truth from persisted state and recovers stranded children + * (see ``handlers/reconcile-stranded-orchestrations.ts``). + * + * Mirrors ``StrandedTaskReconciler``. Grants match the live reconciler + * because it runs the same ``createTaskCore`` release path in-process. + */ + +/** Sweep Lambda timeout (minutes) — matches the live reconciler's createTaskCore + * + Bedrock/S3 SDK bundle cold-start + release work. */ +const SWEEP_TIMEOUT_MINUTES = 5; + +export class StrandedOrchestrationReconciler extends Construct { + public readonly fn: lambda.NodejsFunction; + + constructor(scope: Construct, id: string, props: StrandedOrchestrationReconcilerProps) { + super(scope, id); + + const handlersDir = path.join(__dirname, '..', 'handlers'); + + this.fn = new lambda.NodejsFunction(this, 'ReconcilerFn', { + entry: path.join(handlersDir, 'reconcile-stranded-orchestrations.ts'), + handler: 'handler', + runtime: Runtime.NODEJS_24_X, + architecture: Architecture.ARM_64, + timeout: Duration.minutes(SWEEP_TIMEOUT_MINUTES), + // 512 MB to match the live reconciler — same createTaskCore + + // Bedrock/S3 SDK bundle (see OrchestrationReconciler memory note). + memorySize: 512, + environment: { + ORCHESTRATION_TABLE_NAME: props.orchestrationTable.tableName, + TASK_TABLE_NAME: props.taskTable.tableName, + TASK_EVENTS_TABLE_NAME: props.taskEventsTable.tableName, + ...(props.orchestratorFunctionArn && { + ORCHESTRATOR_FUNCTION_ARN: props.orchestratorFunctionArn, + }), + }, + bundling: { + externalModules: ['@aws-sdk/*'], + // Imports createTaskCore (releases children, which can carry attachments) — + // its transitive attachment-screening PDF path needs pdf-parse unbundled. + // Kept in lockstep with the other attachment-screening constructs; see the + // //:check:pdf-parse-bundling guard that enforces this. + nodeModules: ['pdf-parse'], + }, + }); + + props.orchestrationTable.grantReadWriteData(this.fn); + props.taskTable.grantReadWriteData(this.fn); + props.taskEventsTable.grantReadWriteData(this.fn); + + const schedule = props.schedule ?? Duration.minutes(10); + const rule = new events.Rule(this, 'SweepSchedule', { + schedule: events.Schedule.rate(schedule), + }); + rule.addTarget(new targets.LambdaFunction(this.fn)); + + NagSuppressions.addResourceSuppressions(this.fn, [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaBasicExecutionRole is required for CloudWatch Logs access', + }, + { + id: 'AwsSolutions-IAM5', + reason: + 'DynamoDB index/* wildcards generated by CDK grantReadWriteData for the ' + + 'orchestration scan + child-task lookups + createTaskCore write path', + }, + ], true); + } +} diff --git a/cdk/src/constructs/task-table.ts b/cdk/src/constructs/task-table.ts index 1e9c6599f..7702faa92 100644 --- a/cdk/src/constructs/task-table.ts +++ b/cdk/src/constructs/task-table.ts @@ -57,8 +57,11 @@ export interface TaskTableProps { * - UserStatusIndex (PK: user_id, SK: status_created_at) — "my tasks" queries * - StatusIndex (PK: status, SK: created_at) — queue processing, monitoring * - IdempotencyIndex (PK: idempotency_key) — sparse index for dedup + * - LinearIssueIndex (PK: linear_issue_id, SK: created_at) — sparse; resolve a + * Linear issue back to its newest ABCA task + PR (the standalone + * comment trigger) * - JiraIssueIndex (PK: jira_issue_identity, SK: created_at) — sparse index - * for resolving a Jira issue to its newest PR-producing task + * for resolving a Jira issue to its newest PR-producing task (#640) */ export class TaskTable extends Construct { /** @@ -79,6 +82,16 @@ export class TaskTable extends Construct { */ public static readonly IDEMPOTENCY_INDEX = 'IdempotencyIndex'; + /** + * GSI name for resolving a Linear issue → its newest ABCA task + PR. + * PK: linear_issue_id, SK: created_at (newest task wins). Sparse — + * only Linear-origin tasks (which write the top-level ``linear_issue_id`` + * attribute) are projected; GitHub/Slack/API tasks are absent. Powers the + * standalone ``@bgagent`` comment trigger on a plain (non-orchestration) + * issue, where no orchestration row records the issue→PR link. + */ + public static readonly LINEAR_ISSUE_INDEX = 'LinearIssueIndex'; + /** * GSI for resolving a tenant-scoped Jira issue to its newest ABCA task. * PK: jira_issue_identity (`{cloudId}#{issueKey}`), SK: created_at. @@ -104,6 +117,17 @@ export class TaskTable extends Construct { pointInTimeRecoverySpecification: { pointInTimeRecoveryEnabled: props.pointInTimeRecovery ?? true, }, + // NEW_IMAGE stream feeds the orchestration reconciler + // (`OrchestrationReconciler`), which reacts to child tasks reaching + // terminal status to release dependency-unblocked children. This is + // the table's FIRST and only stream consumer — deliberately on + // TaskTable rather than TaskEventsTable, whose stream is already at + // its 2-consumer limit (FanOutConsumer + ApprovalMetricsPublisher; + // see TaskEventsTable). NEW_IMAGE suffices — the reconciler reads + // status/build_passed/orchestration_id off the new record image. + // Enabling a stream on an existing table is an in-place CFN update + // (no table replacement). + stream: dynamodb.StreamViewType.NEW_IMAGE, removalPolicy: props.removalPolicy ?? RemovalPolicy.DESTROY, }); @@ -130,9 +154,28 @@ export class TaskTable extends Construct { projectionType: dynamodb.ProjectionType.KEYS_ONLY, }); - // Sparse: only Jira-origin tasks with issue metadata carry the top-level - // jira_issue_identity attribute. Keep the projection limited to fields the - // comment-trigger resolver needs. + // GSI: Linear issue → newest ABCA task + PR (sparse — only Linear-origin + // tasks carry the top-level linear_issue_id), for the standalone comment + // trigger. INCLUDE-projects just the fields the trigger reads, so + // the index stays lean (no full-item copy on every task write). + this.table.addGlobalSecondaryIndex({ + indexName: TaskTable.LINEAR_ISSUE_INDEX, + partitionKey: { name: 'linear_issue_id', type: dynamodb.AttributeType.STRING }, + sortKey: { name: 'created_at', type: dynamodb.AttributeType.STRING }, + projectionType: dynamodb.ProjectionType.INCLUDE, + // NOTE: a GSI's projection CANNOT be changed in place — DynamoDB rejects it + // ("Cannot update GSI's properties other than Provisioned Throughput…"; + // observed when deploying the iteration reply). So the iteration reply's + // running-total query does a per-task GetItem for cost_usd rather than + // widening this projection. Keep this list as-is unless you create a NEW + // index with a different name. + nonKeyAttributes: ['pr_url', 'pr_number', 'status', 'repo', 'user_id', 'channel_metadata'], + }); + + // GSI: Jira issue → newest PR-producing task (#640 comment-triggered PR + // iteration). Sparse: only Jira-origin tasks with issue metadata carry the + // top-level jira_issue_identity attribute. Same lean INCLUDE projection as + // the Linear index above — just the fields the comment-trigger resolver reads. this.table.addGlobalSecondaryIndex({ indexName: TaskTable.JIRA_ISSUE_INDEX, partitionKey: { name: 'jira_issue_identity', type: dynamodb.AttributeType.STRING }, diff --git a/cdk/src/handlers/orchestration-reconciler.ts b/cdk/src/handlers/orchestration-reconciler.ts new file mode 100644 index 000000000..eb23db6a6 --- /dev/null +++ b/cdk/src/handlers/orchestration-reconciler.ts @@ -0,0 +1,1408 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Orchestration reconciler for a declared parent/sub-issue dependency graph. + * + * Consumes the **TaskTable DynamoDB stream** (sole consumer — TaskTable + * had no stream before this; TaskEventsTable's is at its 2-consumer + * limit, see that construct's note). On each child task that reaches a + * terminal status, it: + * 1. resolves the task's orchestration via the ChildTaskIndex GSI + * (skips non-orchestration tasks — they have no orchestration_id), + * 2. loads the orchestration snapshot, + * 3. computes the gating plan (pure: orchestration-reconcile.ts), + * 4. persists child-status updates and releases newly-unblocked + * children via the shared release helper. + * + * Idempotent: stream redelivery re-runs the same plan; status updates + * are conditional and releaseChild is idempotency-keyed, so a replayed + * terminal event neither double-releases nor regresses state. + */ + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { + BatchGetCommand, + DynamoDBDocumentClient, + GetCommand, + QueryCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; +import type { DynamoDBBatchResponse, DynamoDBRecord, DynamoDBStreamEvent } from 'aws-lambda'; +import { createTaskCore } from './shared/create-task-core'; +import { renderFailureReply, renderPanelFailureReason } from './shared/failure-reply'; +import { sumIterationCostForIssue } from './shared/iteration-cost'; +import { isNoChangeIteration, renderMaturingReply } from './shared/iteration-reply'; +import { claimTerminalReply, releaseReplyClaim } from './shared/iteration-reply-claim'; +import { logger } from './shared/logger'; +import type { Channel, IssueRef } from './shared/orchestration-channel'; +import { channelForSource, type ChannelRegistryTables } from './shared/orchestration-channel-factory'; +import { computeLeaves, isIntegrationNode } from './shared/orchestration-integration-node'; +import { ORCH_LOG } from './shared/orchestration-log-events'; +import { + computeReconcilePlan, + computeRecoveryPlan, + type ReconcileChild, + type TerminalOutcome, +} from './shared/orchestration-reconcile'; +import { applyTerminalCreateFailures, readConcurrencyBudget, releaseReadyChildren } from './shared/orchestration-release'; +import { planDirectRestack, type RestackStep } from './shared/orchestration-restack'; +import { cascadeNodeLabel, upsertEpicPanel } from './shared/orchestration-rollup'; +import { + claimRollup, + clearRollupClaim, + loadOrchestration, + setRetryCommentId, + setStatusCommentId, + type OrchestrationChildRow, +} from './shared/orchestration-store'; +import { encodeMarkdownUrl } from './shared/screenshot-url'; +import { OrchestrationTable } from '../constructs/orchestration-table'; +import { TaskStatus, TERMINAL_STATUSES, type TaskStatusType } from '../constructs/task-status'; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const ORCHESTRATION_TABLE = process.env.ORCHESTRATION_TABLE_NAME!; +const TASK_TABLE = process.env.TASK_TABLE_NAME!; +// Registry table for the parent rollup comment's per-workspace OAuth +// token. Unset → rollup is skipped (gating still works). +const WORKSPACE_REGISTRY_TABLE = process.env.LINEAR_WORKSPACE_REGISTRY_TABLE_NAME; +/** Jira's tenant-credentials registry. Unset until the stack wires it, which is + * why a Jira-sourced orchestration resolves to no adapter (and skips feedback) + * rather than failing every call. */ +const JIRA_REGISTRY_TABLE = process.env.JIRA_WORKSPACE_REGISTRY_TABLE_NAME; + +/** Credentials registry per surface, for picking an adapter from a stored row. + * A surface left unset here simply has no adapter available in this handler. */ +const CHANNEL_REGISTRY_TABLES: ChannelRegistryTables = { + linear: WORKSPACE_REGISTRY_TABLE, + jira: JIRA_REGISTRY_TABLE, +}; + +/** + * The surface adapter all issue feedback goes through, chosen from the + * orchestration's OWN recorded channel rather than assumed. This handler is + * event-driven: it acts on an orchestration it loaded, so the surface is a + * property of that row, not of the trigger that woke us. Returns undefined when + * the row's surface has no adapter or no configured registry — the caller then + * skips feedback instead of addressing the wrong surface. + */ +const channelForMeta = (meta: { release_context?: { channel_source?: string } }): Channel | undefined => + channelForSource(meta.release_context?.channel_source, CHANNEL_REGISTRY_TABLES); + +/** Build the neutral issue reference the channel operates on. */ +const issueRef = (issueId: string, workspaceId: string): IssueRef => ({ + issueId, + credentialsRef: workspaceId, +}); +// createTaskCore rejects idempotency keys longer than this; synthesized keys +// slice to fit the validated /^[A-Za-z0-9_-]{1,128}$/ pattern. +const MAX_IDEMPOTENCY_KEY_LENGTH = 128; +// Throttle releases to the user's free concurrency budget so a wide +// fan-out doesn't over-release children that admission then hard-fails. Unset +// table → no throttle (release-all, back-compat; admission still gates). +const USER_CONCURRENCY_TABLE = process.env.USER_CONCURRENCY_TABLE_NAME; +const MAX_CONCURRENT = Number(process.env.MAX_CONCURRENT_TASKS_PER_USER ?? '10'); +/** + * Terminal task statuses that the reconciler reacts to. + * + * DERIVED from the same constant the stream's ``FilterCriteria`` is built from, + * not restated. The two must agree: the filter decides which records reach this + * handler, and this set decides which of them it acts on. A second literal list + * would let them drift silently — a status added to one but not the other either + * never arrives or arrives and is dropped, and both look like nothing happened. + */ +const TERMINAL: ReadonlySet = new Set(TERMINAL_STATUSES); + +/** A terminal task event extracted from a TaskTable stream record. */ +interface TerminalTaskEvent { + readonly taskId: string; + readonly status: TaskStatusType; + readonly buildPassed?: boolean; + /** Raw agent error_message, if any — drives the failure reply's detail line. */ + readonly errorMessage?: string; + readonly orchestrationId?: string; + /** + * Cascade marker: set when this terminal task is an + * ITERATION or RESTACK on an orchestration node (carries + * ``orchestration_sub_issue_id`` in channel_metadata but is NOT itself a + * child-row task — its task_id isn't a ``child_task_id``). On COMPLETED we + * re-stack that node's DIRECT dependents. The marker is set by the comment + * trigger (pr-iteration) and by restack tasks themselves (so a restack's + * completion cascades the next hop). + */ + readonly cascadeSubIssueId?: string; + /** + * True when the cascade source was an ITERATION (a human @bgagent comment), + * vs a restack (a predecessor-change ripple). Drives the panel's "updating + * per 's comment" vs "updating to include 's change" phrasing. + */ + readonly cascadeIsIteration?: boolean; + /** + * The Linear comment id that triggered this iteration (set only + * for iterations — a human @bgagent comment). When the iteration task lands, + * the reconciler posts a threaded ✅/❌ reply BENEATH this comment, closing + * the conversation the human opened. Absent on restack cascades (no human + * comment to reply to). + */ + readonly triggerCommentId?: string; + /** + * The Linear ISSUE the trigger comment lives on. Usually the + * iterated sub-issue, but for a comment left on the PARENT epic (which the + * processor routes to a sub-issue) it's the PARENT issue id. The reply must + * use THIS as commentCreate's issueId — Linear rejects a reply whose parentId + * belongs to a different issue. Absent on older tasks → reply falls back to + * the sub-issue id (the prior behavior). + */ + readonly triggerCommentIssueId?: string; + /** + * Whether this iteration advanced the PR branch (a real commit) vs. + * ran with no change (a question-only comment). ``undefined`` for older + * tasks / non-iterations → the success reply defaults to "✅ Updated". + */ + readonly codeChanged?: boolean; + /** The agent's answer, surfaced on a no-change iteration reply. */ + readonly answerText?: string; + /** + * The maturing "👀 On it" reply posted at trigger time. When + * present, the settle EDITS this reply (👀→✅/💬) instead of posting a fresh + * one. Absent on older tasks → falls back to a new threaded reply. + */ + readonly iterationReplyId?: string; + /** This iteration's cost (USD) — folded into the settle reply. */ + readonly costUsd?: number; + /** This iteration's wall-clock seconds — folded into the reply. */ + readonly durationS?: number; +} + +/** + * Extract a terminal-task event from a TaskTable stream record. Returns + * null for records we don't act on (inserts, non-terminal MODIFYs, + * non-orchestration tasks, malformed images). + */ +export function parseTerminalTaskRecord(record: DynamoDBRecord): TerminalTaskEvent | null { + if (record.eventName !== 'MODIFY' && record.eventName !== 'INSERT') return null; + const img = record.dynamodb?.NewImage; + if (!img) return null; + + const taskId = img.task_id?.S; + const status = img.status?.S as TaskStatusType | undefined; + if (!taskId || !status) return null; + if (!TERMINAL.has(status)) return null; + + // Only orchestration children carry orchestration_id. Non-orchestration + // tasks stream through here too (single consumer on the whole table) — + // skip them cheaply. + // + // createTaskCore persists channel metadata as a nested ``channel_metadata`` + // MAP, NOT as a top-level attribute — so read orchestration_id from there. + // (A top-level ``orchestration_id`` exists on the TaskRecord type for + // future use, but createTaskCore doesn't populate it from channel context; + // releaseChild threads the id via channelMetadata.orchestration_id.) + const orchestrationId = + img.orchestration_id?.S + ?? img.channel_metadata?.M?.orchestration_id?.S; + if (!orchestrationId) return null; + + const buildPassed = img.build_passed?.BOOL; + const errorMessage = img.error_message?.S; + // Did this iteration commit anything, and (if not) what did the agent say? + const codeChanged = img.code_changed?.BOOL; + const answerText = img.answer_text?.S; + // The maturing reply to edit + this run's cost/duration. + const iterationReplyId = img.channel_metadata?.M?.iteration_reply_comment_id?.S; + const costUsd = img.cost_usd?.N !== undefined ? Number(img.cost_usd.N) + : (img.cost_usd?.S !== undefined ? Number(img.cost_usd.S) : undefined); + const durationS = img.duration_s?.N !== undefined ? Number(img.duration_s.N) + : (img.duration_s?.S !== undefined ? Number(img.duration_s.S) : undefined); + + // Cascade marker: an iteration/restack task names the node it acted on + // via channel_metadata. A restack task also carries + // ``restack_predecessor_sub_issue_id`` — its presence (or the explicit + // ``orchestration_iteration`` flag the comment trigger sets) marks this as + // a cascade SOURCE rather than a normal child task. We resolve the acted-on + // node from ``orchestration_sub_issue_id`` and confirm "is this a child row?" + // in the handler (a child-row task drives normal gating; a non-child-row + // task with this marker drives the cascade). + const cm = img.channel_metadata?.M; + const isIteration = cm?.orchestration_iteration?.S === 'true'; + const isCascadeSource = + cm?.restack_predecessor_sub_issue_id?.S !== undefined || isIteration; + const cascadeSubIssueId = isCascadeSource ? cm?.orchestration_sub_issue_id?.S : undefined; + // The human comment that triggered this iteration, if any. + const triggerCommentId = isIteration ? cm?.trigger_comment_id?.S : undefined; + // The issue that comment lives on (the parent epic for a parent-routed + // comment; the sub-issue for a direct comment). + const triggerCommentIssueId = isIteration ? cm?.trigger_comment_issue_id?.S : undefined; + + return { + taskId, + status, + ...(buildPassed !== undefined && { buildPassed }), + ...(errorMessage !== undefined && { errorMessage }), + orchestrationId, + ...(cascadeSubIssueId !== undefined && { cascadeSubIssueId }), + ...(cascadeSubIssueId !== undefined && { cascadeIsIteration: isIteration }), + ...(triggerCommentId !== undefined && { triggerCommentId }), + ...(triggerCommentIssueId !== undefined && { triggerCommentIssueId }), + ...(codeChanged !== undefined && { codeChanged }), + ...(answerText !== undefined && { answerText }), + ...(iterationReplyId !== undefined && { iterationReplyId }), + ...(costUsd !== undefined && Number.isFinite(costUsd) && { costUsd }), + ...(durationS !== undefined && Number.isFinite(durationS) && { durationS }), + }; +} + +/** + * Resolve the sub_issue_id for a terminal task within its orchestration. + * Prefers the ChildTaskIndex GSI (task_id → row); the orchestration_id on + * the task record is the authoritative grouping. + */ +async function resolveSubIssueId(taskId: string): Promise { + const res = await ddb.send(new QueryCommand({ + TableName: ORCHESTRATION_TABLE, + IndexName: OrchestrationTable.CHILD_TASK_INDEX, + KeyConditionExpression: 'child_task_id = :tid', + ExpressionAttributeValues: { ':tid': taskId }, + Limit: 1, + })); + const item = res.Items?.[0] as OrchestrationChildRow | undefined; + return item?.sub_issue_id ?? null; +} + +/** + * Batch-read each child's PR url from the TaskTable for the final rollup. + * pr_url lands on the TaskRecord in a separate write from the + * status transition, so it is not on the orchestration row — but by the + * time the orchestration is all-terminal the PRs have settled, so a read + * here is reliable. Best-effort: a failed/partial read just yields fewer + * links (never throws out of the reconcile). Returns ``sub_issue_id → pr_url``. + */ +async function resolveChildPrUrls( + children: readonly OrchestrationChildRow[], +): Promise> { + const withTask = children.filter((c) => c.child_task_id); + if (withTask.length === 0) return {}; + const taskToSub = new Map(withTask.map((c) => [c.child_task_id!, c.sub_issue_id])); + const keys = [...taskToSub.keys()].map((task_id) => ({ task_id })); + const out: Record = {}; + try { + // BatchGet caps at 100 keys/request; an orchestration is far smaller, + // but chunk defensively so a large epic never throws on the limit. + for (let i = 0; i < keys.length; i += 100) { + const chunk = keys.slice(i, i + 100); + const res = await ddb.send(new BatchGetCommand({ + RequestItems: { [TASK_TABLE]: { Keys: chunk, ProjectionExpression: 'task_id, pr_url' } }, + })); + for (const rec of res.Responses?.[TASK_TABLE] ?? []) { + const taskId = rec.task_id as string | undefined; + const prUrl = rec.pr_url as string | undefined; + const sub = taskId ? taskToSub.get(taskId) : undefined; + if (sub && prUrl) out[sub] = prUrl; + } + } + } catch (err) { + logger.warn('Rollup pr_url batch-read failed (non-fatal) — rollup posts without links', { + error: err instanceof Error ? err.message : String(err), + }); + } + return out; +} + +/** + * Batch-read each FAILED child's failure detail from the + * TaskTable so the panel can render WHY it failed + WHERE to read it. Mirrors + * {@link resolveChildPrUrls}: ``error_message`` / ``build_passed`` land on the + * TaskRecord (not the orchestration row), and by the time the epic settles + * they've been written. Only failed children with a task id are read. Composes + * the one-line reason via {@link renderPanelFailureReason}, tagging the + * synthetic integration node so its copy names the combined merge build — the + * exact failure that was previously surfaced as a bare "❌ … failed". + * Best-effort: a read miss just yields no sub-line (never throws out of the + * reconcile). Returns ``sub_issue_id → reason``. + */ +async function resolveChildFailureReasons( + children: readonly OrchestrationChildRow[], +): Promise> { + // A child that failed WITHOUT ever getting a task (deterministic create + // failure — guardrail/validation) carries its reason on the row's + // ``failure_reason`` instead of a task record. Seed the map with those first + // so the panel ❌ has a "why + how to fix" line even though there's no task. + const out: Record = {}; + for (const c of children) { + if (c.child_status === 'failed' && !c.child_task_id && c.failure_reason) { + out[c.sub_issue_id] = c.failure_reason; + } + } + const failed = children.filter((c) => c.child_status === 'failed' && c.child_task_id); + if (failed.length === 0) return out; + const taskToSub = new Map(failed.map((c) => [c.child_task_id!, c.sub_issue_id])); + const isIntegration = new Map(failed.map((c) => [c.sub_issue_id, isIntegrationNode(c.sub_issue_id)])); + const keys = [...taskToSub.keys()].map((task_id) => ({ task_id })); + try { + for (let i = 0; i < keys.length; i += 100) { + const chunk = keys.slice(i, i + 100); + const res = await ddb.send(new BatchGetCommand({ + RequestItems: { + [TASK_TABLE]: { Keys: chunk, ProjectionExpression: 'task_id, error_message, build_passed' }, + }, + })); + for (const rec of res.Responses?.[TASK_TABLE] ?? []) { + const taskId = rec.task_id as string | undefined; + const sub = taskId ? taskToSub.get(taskId) : undefined; + if (!sub || !taskId) continue; + const reason = renderPanelFailureReason({ + ...(typeof rec.build_passed === 'boolean' && { buildPassed: rec.build_passed as boolean }), + ...(typeof rec.error_message === 'string' && { errorMessage: rec.error_message as string }), + taskId, + isIntegration: isIntegration.get(sub) ?? false, + }); + if (reason) out[sub] = reason; + } + } + } catch (err) { + logger.warn('Panel failure-reason batch-read failed (non-fatal) — panel posts without sub-lines', { + error: err instanceof Error ? err.message : String(err), + }); + } + return out; +} + +/** + * The single leaf of a graph that has exactly one, or undefined when the graph + * has several leaves (an integration node covers that case) or none. + * + * Reuses {@link computeLeaves} rather than re-deriving "has no successor" here, + * so the panel's idea of the final node cannot drift from the seeder's. + */ +function soleLeafChild( + children: readonly OrchestrationChildRow[], +): OrchestrationChildRow | undefined { + const leaves = computeLeaves(children.map((c) => ({ + id: c.sub_issue_id, + depends_on: [...c.depends_on], + title: c.sub_issue_id, + }))); + if (leaves.length !== 1) return undefined; + return children.find((c) => c.sub_issue_id === leaves[0]); +} + +/** + * Read the integration node's deploy-preview screenshot URL from its + * TaskRecord (persisted by the screenshot pipeline) so the parent panel can + * embed the combined preview. Best-effort — null when the node has no task, + * no preview deployed yet, or the read fails. Only the integration node is + * read (one Get), since that's the only node whose preview is "combined". + */ +async function resolveCombinedScreenshotUrl( + taskId?: string, +): Promise<{ url: string; previewUrl?: string } | null> { + if (!taskId) return null; + try { + const res = await ddb.send(new GetCommand({ + TableName: TASK_TABLE, + Key: { task_id: taskId }, + ProjectionExpression: 'screenshot_url, screenshot_preview_url', + })); + const url = res.Item?.screenshot_url; + if (typeof url !== 'string' || url.length === 0) return null; + const previewUrl = res.Item?.screenshot_preview_url; + // The live preview-deploy URL makes the panel's combined + // preview a clickable deep-link to the running combined site. + return { + url, + ...(typeof previewUrl === 'string' && previewUrl.length > 0 && { previewUrl }), + }; + } catch (err) { + logger.warn('Combined screenshot read failed (non-fatal) — panel posts without it', { + task_id: taskId, error: err instanceof Error ? err.message : String(err), + }); + return null; + } +} + +/** + * Strongly-consistent re-read of the iteration task's screenshot + + * deploy URL, taken right before the settle renders. Mirrors the fanout + * (standalone) path's ``reloadScreenshotFields``: the screenshot webhook persists + * ``screenshot_url`` onto this task durably but AFTER the deploy, so a non- + * consistent read (or relying only on the comment-append convergence) can miss + * it and the terminal-settle re-render then clobbers the preview — a defect the + * standalone path already fixed this way. ConsistentRead beats the lag. Returns + * nulls on any failure (best-effort). Caller renders the thumbnail when present. + */ +async function reloadIterationScreenshot(taskId?: string): Promise<{ screenshotUrl: string | null; deployUrl: string | null }> { + if (!taskId) return { screenshotUrl: null, deployUrl: null }; + try { + const res = await ddb.send(new GetCommand({ + TableName: TASK_TABLE, + Key: { task_id: taskId }, + ProjectionExpression: 'screenshot_url, screenshot_preview_url', + ConsistentRead: true, + })); + const s = res.Item?.screenshot_url; + const d = res.Item?.screenshot_preview_url; + return { + screenshotUrl: typeof s === 'string' && s.length > 0 ? s : null, + deployUrl: typeof d === 'string' && d.length > 0 ? d : null, + }; + } catch (err) { + logger.warn('Iteration screenshot re-read failed (non-fatal)', { + task_id: taskId, error: err instanceof Error ? err.message : String(err), + }); + return { screenshotUrl: null, deployUrl: null }; + } +} + +/** Apply one terminal child's reconcile plan. */ +async function reconcileTerminalChild(evt: TerminalTaskEvent): Promise { + const orchestrationId = evt.orchestrationId!; + + const subIssueId = await resolveSubIssueId(evt.taskId); + if (!subIssueId) { + logger.warn('Reconciler could not resolve sub_issue_id for terminal task', { + task_id: evt.taskId, + orchestration_id: orchestrationId, + }); + return; + } + + const snapshot = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + if (!snapshot) { + logger.warn('Reconciler found no orchestration snapshot (TTL-reaped?)', { + orchestration_id: orchestrationId, + task_id: evt.taskId, + }); + return; + } + + const children: ReconcileChild[] = snapshot.children.map((c) => ({ + sub_issue_id: c.sub_issue_id, + depends_on: c.depends_on, + child_status: c.child_status, + })); + + const outcome: TerminalOutcome = { + sub_issue_id: subIssueId, + status: evt.status as TerminalOutcome['status'], + ...(evt.buildPassed !== undefined && { build_passed: evt.buildPassed }), + }; + + const plan = computeReconcilePlan(outcome, children); + const now = new Date().toISOString(); + + // 1. Persist status updates (terminal child + any skips). Each is + // conditional on the row not already being in the target state so a + // replayed event is a no-op. + for (const update of plan.statusUpdates) { + // ``toRelease`` rows are handled by releaseChild below (which flips + // them to released conditionally); skip them here to avoid a + // double-write race. + if (plan.toRelease.includes(update.sub_issue_id)) continue; + // A `skipped` transition must only apply from a non-live source + // (blocked/ready). Without this, a skip cascade racing a recovery re-release + // could stamp a `released`/`releasing`/running child terminal-`skipped`, + // letting the epic claim its once-only rollup while work is still in flight. + // Other resets (failed→ready/blocked) keep the plain not-already-there guard. + const skipGuard = update.child_status === 'skipped'; + const values: Record = { ':s': update.child_status, ':now': now }; + let condition = 'child_status <> :s'; + if (skipGuard) { + values[':blocked'] = 'blocked'; + values[':ready'] = 'ready'; + condition = 'child_status IN (:blocked, :ready)'; + } + try { + await ddb.send(new UpdateCommand({ + TableName: ORCHESTRATION_TABLE, + Key: { orchestration_id: orchestrationId, sub_issue_id: update.sub_issue_id }, + UpdateExpression: 'SET child_status = :s, updated_at = :now', + ConditionExpression: condition, + ExpressionAttributeValues: values, + })); + } catch (err) { + if (isConditionalCheckFailed(err)) continue; // already in target / not a skippable source + throw err; + } + } + + // 1b. Reconcile a FAILED child's OWN issue state on the surface. The agent + // moves a writeable child to awaiting-review only on AGENT success; but the + // platform build gate is independent — a child can finish COMPLETED with + // build_passed=false (PR opened, build red), and a child that succeeded on + // an EARLIER run (→ awaiting-review) then fails a RETRY leaves that state + // behind (the agent's failure path leaves state as-is; transitions never + // move backward). Either way the graph says `failed` while the issue still + // reads awaiting-review with a PR link — a visible contradiction. The + // reconciler holds the authoritative verdict, so when a terminal child did + // NOT succeed we pull its issue back out of any bot-set running state. + // `revertState` is tightly guarded per surface (only demotes an issue still + // in a bot-set running state — never a human-advanced done/canceled or a + // human-pulled-back one) and idempotent on replay. The ❌ reaction + the + // failure reason on the panel convey "it failed"; a reply re-runs it (the + // retry re-drives the state not-started → running). Skip the integration + // node (synthetic id, no real issue). Best-effort; never throws. + const settleChannel = channelForMeta(snapshot.meta); + if (settleChannel && !plan.terminalSucceeded && !isIntegrationNode(subIssueId)) { + const channel = settleChannel; + const childRef = issueRef(subIssueId, snapshot.meta.credentials_ref); + try { + const reverted = (await channel.revertState?.(childRef)) ?? false; + // Also settle the child's ISSUE reaction to ❌. The agent reacts ✅ on its + // OWN verdict (agent-success + regression-only build gate — a build that was + // already red before the agent isn't counted as the agent's regression, so + // it posts ✅ and "Task completed"). The orchestration gate is stricter + // (build_passed===false ⇒ failed, absolute), so a child can legitimately end + // agent-✅ but graph-failed — leaving a ✅ reaction that contradicts the + // failed node (observed in practice: PR opened, agent ✅, build red). + // Replacing the reaction clears only the bot's own status markers and adds + // ❌ — a human's reaction is never touched, and it's idempotent on replay. + // This makes the reaction agree with the reverted state + the panel's ❌ + // row. (The stale "✅ Task completed" comment is left as history — the + // surface won't let us edit another actor's comment; the reaction + state + + // panel are the authoritative signals, and a reply re-runs the child.) + const reactionSwapped = (await channel.replaceIssueReaction?.(childRef, 'failed')) ?? false; + if (reverted || reactionSwapped) { + logger.info('Reconciler settled a failed child to match the graph (state + reaction)', { + orchestration_id: orchestrationId, + sub_issue_id: subIssueId, + task_status: outcome.status, + build_passed: outcome.build_passed, + state_reverted: reverted, + reaction_swapped: reactionSwapped, + }); + } + } catch (err) { + logger.warn('Failed to reconcile failed child Linear state/reaction (non-fatal)', { + orchestration_id: orchestrationId, + sub_issue_id: subIssueId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + // 2. Re-evaluate releasability against a FRESH read, not the initial + // snapshot. + // + // Concurrency (failure-matrix row 3): when two predecessors of the + // same child D finish simultaneously, each reconciler invocation + // loads its own snapshot, persists only ITS child as succeeded, and + // — working from its stale snapshot — sees D's OTHER predecessor not + // yet succeeded, so neither releases D and it strands ``blocked``. + // The plan's ``toRelease`` (computed from the initial snapshot) is + // therefore unreliable under concurrency. Reloading after the + // status write means whichever invocation reads last sees BOTH + // predecessors succeeded and releases D; the conditional + // ready→released flip in releaseChild dedups if both happen to see it. + const fresh = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + // Reassignable so a terminal create-failure can be patched into the view + // before the settle below (see applyTerminalCreateFailures). + let freshChildren = fresh?.children ?? snapshot.children; + const succeeded = new Set( + freshChildren.filter((c) => c.child_status === 'succeeded').map((c) => c.sub_issue_id), + ); + const releasableRows = freshChildren + .filter((c) => + // newly-unblocked: all predecessors now succeeded + (c.child_status === 'blocked' && c.depends_on.every((d) => succeeded.has(d))) + // OR still `ready` and un-released: either throttle-deferred (a prior + // pass left it ready when the concurrency budget was full) OR reset-to-ready + // by an epic retry (which KEEPS its prior child_task_id so the + // idempotency salt spawns a fresh task). We deliberately do NOT gate on + // !child_task_id: that gate stranded retried children forever, and the + // flip-then-create claim (ready→releasing) is now the race guard — + // only one releaser wins, so re-picking a ready-with-id child is safe. Roots + // have no predecessors so depends_on.every(...) is vacuously true. + || (c.child_status === 'ready' && c.depends_on.every((d) => succeeded.has(d)))) + .map((c) => ({ ...c, child_status: 'ready' as const })); + + if (releasableRows.length > 0) { + const releaseCtx = (fresh ?? snapshot).meta.release_context; + // Throttle this pass to the user's free concurrency budget so a + // wide fan-out doesn't over-release children that admission then + // hard-fails (the cap is a throttle, not a guillotine). Leftover ready + // children are released by the next reconcile (a sibling completing + // re-fires this handler) or the stranded-orchestration sweep, as slots + // free. Unset table → release all (back-compat; admission still gates). + const budget = USER_CONCURRENCY_TABLE + ? await readConcurrencyBudget(ddb, USER_CONCURRENCY_TABLE, releaseCtx.platform_user_id, MAX_CONCURRENT) + : undefined; + const results = await releaseReadyChildren( + ddb, + ORCHESTRATION_TABLE, + releasableRows, + releaseCtx, + createTaskCore, + now, + // Pass the full child set so each releasable child's base + // branch can be derived from its predecessors' persisted branches. + freshChildren, + 'main', + budget, + ); + logger.info('Reconciler released children', { + orchestration_id: orchestrationId, + trigger_sub_issue_id: subIssueId, + released: results.filter((r) => r.kind === 'released').length, + terminally_failed: results.filter((r) => r.kind === 'create_failed_terminal').length, + requested: releasableRows.length, + ...(budget !== undefined && { concurrency_budget: budget }), + }); + // A child that failed DETERMINISTICALLY (guardrail/validation) was just + // marked terminally 'failed' in the store, but our in-memory freshChildren + // still shows it releasing/ready — and no later stream event will re-drive + // this reconciler for a child that never became a task. Persist the + // transitive skips of its dependents + patch the local view so the settle + // below sees the full terminal picture and can reach all-terminal + post ❌. + freshChildren = await applyTerminalCreateFailures( + ddb, ORCHESTRATION_TABLE, orchestrationId, freshChildren, results, now, + ); + } + + // Refresh the panel + settle the parent state against the fresh view. + await refreshPanelAndSettle(orchestrationId, freshChildren, (fresh ?? snapshot).meta, now); +} + +/** + * Maintain the SINGLE maturing epic panel — one comment, edited in + * place — and settle the parent state when the epic reaches all-terminal. + * Shared by the normal child-gating path (``reconcileTerminalChild``) AND the + * cascade path (``cascadeRestack``): a re-stack/iteration task completing must + * ALSO clear its node's ``🔄 updating`` row and re-run the completion check, or + * an epic whose only remaining activity is a cascade hangs forever at + * "🔄 N/M" with a stale updating row (observed in practice — a re-stack of a + * no-dependents node returned early and never refreshed). + * + * Best-effort; only when the orchestration's surface has a usable adapter. The + * panel BODY edit is idempotent (same body = no-op), so it always runs; the + * parent-STATE mirror is claimed once via ``claimRollup`` on the first + * all-terminal caller. + */ +export async function refreshPanelAndSettle( + orchestrationId: string, + children: readonly OrchestrationChildRow[], + meta: { + credentials_ref: string; + parent_issue_ref: string; + status_comment_id?: string; + retry_comment_id?: string; + release_context?: { channel_source?: string; trigger_label?: string }; + }, + now: string, +): Promise { + const channel = channelForMeta(meta); + if (!channel) return; + + // Completion check: every child terminal (succeeded/failed/skipped — + // released is NOT terminal). + const allTerminal = children.every((c) => + c.child_status === 'succeeded' || c.child_status === 'failed' || c.child_status === 'skipped', + ); + + const prUrls = await resolveChildPrUrls(children); + // When any node failed, resolve its one-line reason + CloudWatch pointer + // so the panel row carries a diagnostic sub-line (the integration node's + // combined-build failure has no other surface). Only read on a failure — + // healthy epics skip the extra BatchGet. + const anyFailed = children.some((c) => c.child_status === 'failed'); + const failureReasons = anyFailed ? await resolveChildFailureReasons(children) : {}; + const integration = children.find((c) => isIntegrationNode(c.sub_issue_id)); + const combinedPrUrl = integration ? prUrls[integration.sub_issue_id] : undefined; + // The node whose deploy preview represents the WHOLE epic. + // + // With several leaves that is the synthetic integration node, which exists to + // merge them. With ONE leaf there is no integration node — by then everything + // has already converged on that final node, so a synthetic one would re-run + // work it had just done and its "combined" preview would duplicate the leaf's + // own. The final node IS the combined result. + // + // Either way the panel should carry a preview: a reviewer reads the parent, and + // "no integration node" is not a reason to show them nothing when a finished + // node holds exactly the artifact they want. + const previewNode = integration ?? soleLeafChild(children); + // Only read on the all-terminal settle (the node has deployed by then); skip + // the extra Get on every in-flight panel edit. + const combinedScreenshot = (allTerminal && previewNode) + ? await resolveCombinedScreenshotUrl(previewNode.child_task_id) + : null; + + if (allTerminal) { + logger.info('Orchestration complete', { + event: ORCH_LOG.orchestrationComplete, + orchestration_id: orchestrationId, + parent_linear_issue_id: meta.parent_issue_ref, + succeeded: children.filter((c) => c.child_status === 'succeeded').length, + failed: children.filter((c) => c.child_status === 'failed').length, + skipped: children.filter((c) => c.child_status === 'skipped').length, + }); + } + + // Idempotency for the PARENT-STATE mirror: the orchestration can reach "all + // terminal" on more than one stream event. Mirror only once, on the first + // all-terminal caller. The panel BODY edit is naturally idempotent. + const won = !allTerminal || await claimRollup(ddb, ORCHESTRATION_TABLE, orchestrationId, now); + + const newId = await upsertEpicPanel({ + channel, + parent: issueRef(meta.parent_issue_ref, meta.credentials_ref), + ...(meta.status_comment_id !== undefined && { statusCommentId: meta.status_comment_id }), + children, + prUrls, + ...(Object.keys(failureReasons).length > 0 && { failureReasons }), + ...(combinedPrUrl !== undefined && { combinedPrUrl }), + ...(combinedScreenshot !== null && { combinedScreenshotUrl: combinedScreenshot.url }), + ...(combinedScreenshot?.previewUrl !== undefined && { combinedPreviewUrl: combinedScreenshot.previewUrl }), + inProgress: !allTerminal, + mirrorParentState: allTerminal ? won : false, + ...(meta.release_context?.trigger_label !== undefined + && { labelFilter: meta.release_context.trigger_label }), + }); + // Persist a freshly-created panel comment id so later edits reuse it. + if (newId && !meta.status_comment_id) { + try { + await setStatusCommentId(ddb, ORCHESTRATION_TABLE, orchestrationId, newId); + } catch (err) { + logger.warn('Failed to persist panel comment id (non-fatal)', { + orchestration_id: orchestrationId, error: err instanceof Error ? err.message : String(err), + }); + } + } + + // Settle the `@bgagent retry` comment that asked for this run, if one did. + // The retry path can only ack it (👀) — it finishes the moment the work is + // dispatched — so the comment the user is watching has no outcome unless this + // settle gives it one. Gated on ``won`` so it happens once per settle, the same + // way the parent-state mirror is. + if (allTerminal && won && meta.retry_comment_id) { + await settleRetryComment(orchestrationId, meta.retry_comment_id, children, channel, meta); + } +} + +/** + * Move a retry comment's marker from the receipt 👀 to the run's outcome. + * + * Mirrors what an iteration's settle does for its trigger comment: the marker is + * the whole answer here, because the retry path posts no maturing reply. ✅ when + * everything ended up succeeding, ❌ when anything is still failed or skipped — + * matching the panel header the user reads directly above it. + * + * The record is cleared FIRST. If the marker swap were to succeed and the clear + * then fail, a later settle of the same epic would re-swap a marker on a comment + * that has already been answered; clearing first means a failure leaves the marker + * un-updated instead, which is the same state as before this ran and is visibly + * wrong rather than silently stale. Best-effort throughout: this is feedback. + */ +async function settleRetryComment( + orchestrationId: string, + retryCommentId: string, + children: readonly OrchestrationChildRow[], + channel: Channel, + meta: { credentials_ref: string; parent_issue_ref: string }, +): Promise { + const anyBad = children.some((c) => c.child_status === 'failed' || c.child_status === 'skipped'); + try { + await setRetryCommentId(ddb, ORCHESTRATION_TABLE, orchestrationId, undefined); + await channel.replaceCommentReaction?.( + { commentId: retryCommentId }, + issueRef(meta.parent_issue_ref, meta.credentials_ref), + anyBad ? 'failed' : 'succeeded', + ); + logger.info('Settled the retry comment to match the epic outcome', { + orchestration_id: orchestrationId, + comment_id: retryCommentId, + outcome: anyBad ? 'failed' : 'succeeded', + }); + } catch (err) { + logger.warn('Could not settle the retry comment (non-fatal)', { + orchestration_id: orchestrationId, + comment_id: retryCommentId, + error: err instanceof Error ? err.message : String(err), + }); + } +} + +/** + * Restack cascade. A terminal ITERATION or RESTACK task on node X + * just completed — re-stack X's DIRECT dependents so they pick up X's new + * branch. Each dependent's own restack completion re-fires this handler and + * cascades the next hop (see ``planDirectRestack``). Only on COMPLETED — a + * failed iteration leaves dependents on the prior (still-valid) base. + * + * Idempotent: the per-dependent task's idempotency key includes the SOURCE + * task id, so the same completion never spawns a dependent's restack twice; + * a different source (the next real change) gets a new key. Best-effort — + * a failure to spawn one dependent does not block the others. + */ +async function cascadeRestack(evt: TerminalTaskEvent): Promise { + const orchestrationId = evt.orchestrationId!; + const changedSubIssueId = evt.cascadeSubIssueId!; + const succeeded = evt.status === TaskStatus.COMPLETED && evt.buildPassed !== false; + const now = new Date().toISOString(); + + // An ITERATION carries the human comment that triggered it. When + // it lands — success OR failure — reply ✅/❌ in a thread beneath that + // comment, closing the conversation the human opened. This runs regardless + // of whether there are dependents to re-stack (a leaf node has none) and + // before the success-gate below (a failed iteration still gets its ❌ reply). + if (evt.triggerCommentId) { + await replyToIterationComment(evt, changedSubIssueId, succeeded); + } + + // Only a successful change should cascade onto dependents. + if (!succeeded) { + logger.info('Restack cascade: source task not successful — not cascading', { + orchestration_id: orchestrationId, + changed_sub_issue_id: changedSubIssueId, + status: evt.status, + }); + return; + } + + const snapshot = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + if (!snapshot) { + logger.warn('Restack cascade: orchestration snapshot not found', { orchestration_id: orchestrationId }); + return; + } + + // RECOVERY cascade. If this successful iteration was a fix on a + // node that is currently ``failed`` (a human commented a fix on a ❌ sub-issue), + // un-fail it and re-release the dependents that were transitively ``skipped`` + // when it first failed — so the WHOLE epic can recover, not just this one PR. + // No-ops cleanly when the node wasn't failed (the normal forward cascade below + // handles a healthy iteration's dependents). + await maybeRecoverFailedNode(orchestrationId, snapshot, changedSubIssueId, now); + + const steps = planDirectRestack(snapshot.children, changedSubIssueId); + if (steps.length === 0) { + logger.info('Restack cascade: no started direct dependents to re-stack', { + orchestration_id: orchestrationId, + changed_sub_issue_id: changedSubIssueId, + }); + // The cascade source (this re-stack/iteration) itself just completed and + // carried a '🔄 updating' row on the panel. With no dependents to ripple + // to, NOTHING else will fire for this node — so we MUST refresh here to + // clear its updating row and re-run the completion check. Without this, an + // epic whose only remaining activity is a leaf-node re-stack hangs forever + // at "🔄 N/M" with a stale updating row (observed in practice). + // Re-load so the panel reflects this node's freshly-persisted terminal + // status, then settle. + const fresh = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + await refreshPanelAndSettle(orchestrationId, (fresh ?? snapshot).children, (fresh ?? snapshot).meta, now); + return; + } + + logger.info('Restack cascade: re-stacking direct dependents', { + orchestration_id: orchestrationId, + changed_sub_issue_id: changedSubIssueId, + source_task_id: evt.taskId, + dependent_count: steps.length, + }); + + // Human-readable label for the changed node (the predecessor that was + // revised), used in the surfacing comments. Prefer its Linear identifier. + const meta = snapshot.meta; + const changedRow = snapshot.children.find((c) => c.sub_issue_id === changedSubIssueId); + // Friendly short name — for the integration node this is "the integration", + // NOT its raw synthetic title (which read clumsily in the possessive cascade + // reason "…'s change" — observed in practice). + const changedLabel = cascadeNodeLabel(changedSubIssueId, changedRow?.display_id, changedRow?.title); + + const cascadeChannel = channelForMeta(meta); + + const updatingIds: string[] = []; + for (const step of steps) { + const created = await spawnRestackTask(step, meta.release_context.platform_user_id, evt.taskId, changedSubIssueId); + // Surface ONLY on a genuinely NEW restack task (201). A 200 means an + // idempotent replay (the cascade source's stream record is redelivered + // multiple times — observed 3× live), so don't re-mark. 'failed' = skip. + if (created !== 'created') continue; + updatingIds.push(step.child.sub_issue_id); + } + + // Instead of standalone '🔄 Re-stacked' / 'revised' comments, + // refresh the SINGLE epic panel so the impacted rows show '🔄 updating per + // ' and the header reverts to in-progress. The dependent's own + // sub-issue gets the react/reply ack, not a status comment here. The + // 'updating' rows settle back to ✅ when their restack tasks complete — those + // completions route to cascadeRestack (NOT reconcileTerminalChild) and clear + // the row via refreshPanelAndSettle (the no-dependents path). + if (cascadeChannel && updatingIds.length > 0) { + // A cascade re-opened an epic that may have ALREADY completed (a comment on + // a finished epic). Release the once-only rollup claim so the parent state + // can re-settle (👀→✅) when the re-stacks finish — else the claim stays + // taken forever and the parent reaction never re-mirrors. + await clearRollupClaim(ddb, ORCHESTRATION_TABLE, orchestrationId, now); + const reason = evt.cascadeIsIteration + ? `per ${changedLabel}'s comment` + : `to include ${changedLabel}'s change`; + const updating: Record = {}; + for (const id of updatingIds) updating[id] = reason; + // Render from a FRESH read, not the pre-spawn snapshot: spawnRestackTask just + // flipped the restacked rows to `released` and stamped branches, so the + // snapshot is stale. The forward-gating path already re-loads after writes; + // mirror that here so the panel reflects current child state (else a stale + // row status shows for one event window). Fall back to the snapshot on a + // read miss. (The `updating` overlay is driven by updatingIds, not statuses.) + const cascadeFresh = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + const panelChildren = cascadeFresh?.children ?? snapshot.children; + const prUrls = await resolveChildPrUrls(panelChildren); + const integration = panelChildren.find((c) => isIntegrationNode(c.sub_issue_id)); + await upsertEpicPanel({ + channel: cascadeChannel, + parent: issueRef(meta.parent_issue_ref, meta.credentials_ref), + ...(meta.status_comment_id !== undefined && { statusCommentId: meta.status_comment_id }), + children: panelChildren, + prUrls, + updating, + ...(integration && prUrls[integration.sub_issue_id] !== undefined + && { combinedPrUrl: prUrls[integration.sub_issue_id] }), + inProgress: true, // a cascade re-opened the epic + }); + } +} + +/** + * RECOVERY cascade. A successful iteration on a node that is + * currently ``failed`` (a human commented a fix on a ❌ sub-issue). Un-fail the + * node and re-release the dependents that were transitively ``skipped`` when it + * first failed, so the whole epic can recover rather than stranding at "finished + * with failures". No-ops when the node isn't failed. + * + * Mirrors {@link reconcileTerminalChild}'s persist-then-release shape: + * 1. {@link computeRecoveryPlan} decides the un-fail + un-skip writes. + * 2. Persist each conditionally (skip the ones release will flip). + * 3. Re-release the freed children via {@link releaseReadyChildren}, honoring + * the user's concurrency budget exactly like the forward path. + * Best-effort + idempotent: a redelivered iteration event finds the node already + * ``succeeded`` (recovery plan empty) and no-ops. + */ +async function maybeRecoverFailedNode( + orchestrationId: string, + snapshot: NonNullable>>, + recoveredSubIssueId: string, + now: string, +): Promise { + const children: ReconcileChild[] = snapshot.children.map((c) => ({ + sub_issue_id: c.sub_issue_id, + depends_on: c.depends_on, + child_status: c.child_status, + })); + const plan = computeRecoveryPlan(recoveredSubIssueId, children); + if (plan.statusUpdates.length === 0) return; // node wasn't failed — nothing to recover + + logger.info('Recovery cascade: un-failing node + resetting skipped subtree', { + orchestration_id: orchestrationId, + recovered_sub_issue_id: recoveredSubIssueId, + un_skipped: plan.statusUpdates.length - 1, + re_releasing: plan.toRelease.length, + }); + + // 1. Persist ALL the un-fail (→succeeded) + un-skip (→blocked) writes, + // INCLUDING the toRelease rows. Unlike the forward path (reconcileTerminalChild), + // we must NOT exclude the toRelease rows here: there they're already + // 'blocked' in the store so releaseReadyChildren can flip them; here they're + // still 'skipped', and releaseReadyChildren's conditional write only accepts + // child_status IN (blocked, ready) — so without first persisting + // skipped→'blocked' the release spawns the task but the row stays 'skipped' + // (observed in practice: a dependent ran + opened a PR yet the panel kept + // showing ⏭️ skipped and the epic never advanced). Persist blocked first, + // then release flips blocked→released. + for (const update of plan.statusUpdates) { + try { + await ddb.send(new UpdateCommand({ + TableName: ORCHESTRATION_TABLE, + Key: { orchestration_id: orchestrationId, sub_issue_id: update.sub_issue_id }, + UpdateExpression: 'SET child_status = :s, updated_at = :now', + ConditionExpression: 'child_status <> :s', + ExpressionAttributeValues: { ':s': update.child_status, ':now': now }, + })); + } catch (err) { + if (isConditionalCheckFailed(err)) continue; + throw err; + } + } + + // 2. The epic had settled to "⚠️ finished with failures" — its rollup claim is + // held and the parent carries the ❌ reaction. Recovery re-opens it: release + // the once-only rollup claim so the parent state can re-settle (❌→🔄→✅) as + // the recovered work lands. Without this the panel + parent reaction stay + // stuck at the failed snapshot even though work is running again (observed + // in practice). + await clearRollupClaim(ddb, ORCHESTRATION_TABLE, orchestrationId, now); + + // 3. Re-release the now-'blocked' freed children against a FRESH read (the + // un-skip writes above must be visible), gated on the concurrency budget + // like the forward path. releaseReadyChildren accepts child_status IN + // (blocked, ready); present them as ready. + const fresh = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + const freshChildren = fresh?.children ?? snapshot.children; + if (plan.toRelease.length > 0) { + const releasableRows = freshChildren + .filter((c) => plan.toRelease.includes(c.sub_issue_id)) + .map((c) => ({ ...c, child_status: 'ready' as const })); + if (releasableRows.length > 0) { + const releaseCtx = (fresh ?? snapshot).meta.release_context; + const budget = USER_CONCURRENCY_TABLE + ? await readConcurrencyBudget(ddb, USER_CONCURRENCY_TABLE, releaseCtx.platform_user_id, MAX_CONCURRENT) + : undefined; + const results = await releaseReadyChildren( + ddb, + ORCHESTRATION_TABLE, + releasableRows, + releaseCtx, + createTaskCore, + now, + freshChildren, + 'main', + budget, + ); + logger.info('Recovery cascade: re-released children', { + orchestration_id: orchestrationId, + recovered_sub_issue_id: recoveredSubIssueId, + released: results.filter((r) => r.kind === 'released').length, + requested: releasableRows.length, + }); + } + } + + // 4. Refresh the panel against the fresh post-recovery view so the un-skipped + // rows stop rendering ⏭️ and the header reverts from "finished with + // failures" to in-progress (the parent reaction re-settles on completion). + const refreshed = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + await refreshPanelAndSettle( + orchestrationId, + (refreshed ?? fresh ?? snapshot).children, + (refreshed ?? fresh ?? snapshot).meta, + now, + ); +} + +/** + * Post the threaded ✅/❌ reply beneath the human ``@bgagent`` + * comment that triggered this iteration. The 👀 reaction already landed (the + * processor's instant ack); this reply closes the loop when the work lands. + * + * Idempotent: the cascade source's stream record is redelivered multiple times + * (observed 3× live), so we claim the right to reply exactly once by + * conditionally stamping ``ack_replied_at`` on the iteration task's own + * TaskTable record (its ``task_id`` is the per-iteration unit). The first + * caller wins and posts; redeliveries lose the conditional write and skip. + * Best-effort throughout — a Linear or DDB hiccup never blocks the cascade. + */ +async function replyToIterationComment( + evt: TerminalTaskEvent, + changedSubIssueId: string, + succeeded: boolean, +): Promise { + const commentId = evt.triggerCommentId!; + + // Resolve the workspace for the reply. The iteration task carries it in + // channel_metadata; rather than re-read the record, load the orchestration + // meta (already cached-cheap) for the workspace id — and the surface to + // answer on, which is the orchestration's own, not an assumed one. + const snapshot = await loadOrchestration(ddb, ORCHESTRATION_TABLE, evt.orchestrationId!); + if (!snapshot) return; + const channel = channelForMeta(snapshot.meta); + if (!channel) return; + const workspaceId = snapshot.meta.credentials_ref; + + // Claim the one reply for this iteration task. + const claim = await claimTerminalReply(ddb, TASK_TABLE, evt.taskId, new Date().toISOString()); + if (!claim.won) return; // lost the claim (replay) or errored → don't double-reply + + // Mature the settle reply (👀→✅/💬) with cost + running total, + // editing the trigger-time reply when its id was captured. A failure keeps the + // standard failure reply (which a human can reply to, to retry). + const prNumber = await resolvePrNumber(evt.taskId); + const prUrl = await resolvePrUrl(evt.taskId); + const runningTotalUsd = (await sumIterationCostForIssue({ + ddb, + taskTableName: TASK_TABLE, + linearIssueId: changedSubIssueId, + thisTaskId: evt.taskId, + ...(evt.costUsd !== undefined && { thisCost: evt.costUsd }), + logLabel: 'reconciler', + })).total; + // Strongly-consistent re-read of this iteration's screenshot so + // the settle renders the preview thumbnail itself (race-free against the + // screenshot webhook's append), matching the fanout/standalone path. Only an + // 'updated' (real edit) state folds the thumbnail in — a question didn't change UI. + const isUpdated = !isNoChangeIteration(evt.codeChanged); + const shot = isUpdated ? await reloadIterationScreenshot(evt.taskId) : { screenshotUrl: null, deployUrl: null }; + const body = succeeded + ? renderMaturingReply({ + state: isNoChangeIteration(evt.codeChanged) ? 'answered' : 'updated', + prNumber, + ...(prUrl !== null && { prUrl }), + ...(evt.answerText !== undefined && { answerText: evt.answerText }), + ...(evt.costUsd !== undefined && { costUsd: evt.costUsd }), + ...(evt.durationS !== undefined && { durationS: evt.durationS }), + ...(runningTotalUsd !== null && { runningTotalUsd }), + ...(shot.screenshotUrl ? { screenshotUrl: shot.screenshotUrl } : {}), + ...(shot.screenshotUrl && shot.deployUrl ? { deployUrl: encodeMarkdownUrl(shot.deployUrl) } : {}), + }) + : renderFailureReply({ + status: evt.status, + buildPassed: evt.buildPassed, + ...(evt.errorMessage !== undefined && { errorMessage: evt.errorMessage }), + taskId: evt.taskId, + }); + // The reply must be addressed to the issue the trigger comment lives on — a + // surface can reject a threaded reply whose parent comment belongs to a + // different issue. For a comment left on the PARENT epic that's the parent + // issue, NOT changedSubIssueId. Fall back to the sub-issue id for tasks + // created before the trigger issue was persisted. + const replyIssueId = evt.triggerCommentIssueId ?? changedSubIssueId; + // EDIT the maturing reply posted at trigger time; fall back to a fresh + // threaded reply for older tasks that captured no reply id. + // preservePreview: converge with the screenshot webhook's async `[preview]` + // append so this terminal re-render doesn't clobber it. + const reply = await channel.upsertThreadedReply?.( + issueRef(replyIssueId, workspaceId), + { commentId }, + body, + evt.iterationReplyId ? { commentId: evt.iterationReplyId } : undefined, + // repairIfOverwritten: a progress render delivered at the same moment can land + // on top of this outcome, and the surface has no conditional update to prevent + // it — so re-assert the outcome if that happened. + { preservePreview: true, repairIfOverwritten: true }, + ); + // A surface that cannot mature a reply at all (the capability is optional) + // legitimately returns undefined; only an attempted-and-failed reply — null — + // means the outcome went unsaid. + if (reply === null) { + // Holding a claim over a reply that never landed makes the failure permanent: + // no redelivery may retry it, and the progress + heartbeat writers read the + // claim as "an outcome has landed" and stand down too, so the reply stays on + // its last progress text. So hand the claim back — but only while attempts + // remain. + const release = await releaseReplyClaim(ddb, TASK_TABLE, evt.taskId, claim.stamp); + if (release !== 'exhausted') { + // A retry is still coming (or another delivery owns the reply). Settle + // nothing yet: a ✅ reaction beside a reply still saying "On it" is the + // contradiction this design exists to remove. + logger.warn('Iteration ack: reply failed — deferring the settle to another attempt', { + task_id: evt.taskId, + linear_issue_id: replyIssueId, + release, + }); + return; + } + // No attempt is coming. Fall through WITHOUT a reply so the reaction on the + // trigger comment still moves off 👀: a reply that will never arrive is + // strictly worse than an outcome shown only as a marker, because 👀 forever + // reads as "still working" — the black box this whole design removes. + logger.error('Iteration ack: giving up on the reply — settling via the reaction alone', { + task_id: evt.taskId, + linear_issue_id: replyIssueId, + }); + } + + // Settle the comment + sub-issue so all three views agree (panel row, + // sub-issue state, comment reaction) — the platform owns this, not the agent + // (whose prompt-driven state-setting flapped between running and in-review). + // - replace the TRIGGER comment's 👀 with ✅ (success) / ❌ (failure), so the + // comment itself reads done at a glance, not just the threaded reply. + // - on success, advance the SUB-ISSUE to awaiting-review (its PR is updated + // & open, awaiting human merge — same convention the epic uses). On + // failure, leave the state (the ❌ + reply convey it). Never demote. + // Best-effort + idempotent (the ack_replied_at claim above already gates this + // to once per iteration; the reaction replace + transition re-converge anyway). + // A no-change iteration (a question) is neither a success-edit nor a failure — + // it's an answer. Don't stamp ✅ (implies "PR updated, merge-worthy") and don't + // advance the sub-issue (nothing changed). Use 💬 and leave the state + // untouched. A real edit keeps the ✅ + awaiting-review convention. + const noChange = succeeded && isNoChangeIteration(evt.codeChanged); + await channel.replaceCommentReaction?.( + { commentId }, + issueRef(replyIssueId, workspaceId), + noChange ? 'needs_input' : (succeeded ? 'succeeded' : 'failed'), + ); + if (succeeded && !noChange) { + await channel.transitionState?.(issueRef(changedSubIssueId, workspaceId), 'in_review'); + } +} + +/** + * Spawn one coding/restack-v1 task for a direct dependent. Best-effort. + * Returns ``'created'`` for a genuinely new task (201), ``'exists'`` for an + * idempotent replay (200 — the source event was redelivered), or ``'failed'``. + * The caller surfaces the re-stack to the user ONLY on ``'created'`` so + * redelivered stream records don't post duplicate comments. + */ +async function spawnRestackTask( + step: RestackStep, + platformUserId: string, + sourceTaskId: string, + changedSubIssueId: string, +): Promise<'created' | 'exists' | 'failed'> { + const child = step.child; + const prNumber = await resolvePrNumber(child.child_task_id); + if (prNumber === null) { + logger.warn('Restack cascade: dependent has no resolvable PR number — skipping', { + orchestration_id: child.orchestration_id, + sub_issue_id: child.sub_issue_id, + child_task_id: child.child_task_id, + }); + return 'failed'; + } + + // Idempotency keyed on the SOURCE task id: this exact completion re-stacks + // a given dependent at most once. Within [A-Za-z0-9_-], ≤128 chars. + const idempotencyKey = `restack_${child.sub_issue_id}_${sourceTaskId}`.replace(/[^A-Za-z0-9_-]/g, '').slice(0, MAX_IDEMPOTENCY_KEY_LENGTH); + + try { + const result = await createTaskCore( + { + repo: child.repo, + workflow_ref: 'coding/restack-v1', + pr_number: prNumber, + }, + { + userId: platformUserId, + channelSource: 'webhook', + channelMetadata: { + orchestration_id: child.orchestration_id, + // This dependent is the next cascade SOURCE: when its restack + // completes, parseTerminalTaskRecord sees restack_predecessor_* + // and cascades to ITS dependents. + orchestration_sub_issue_id: child.sub_issue_id, + restack_predecessor_sub_issue_id: changedSubIssueId, + // repo.py merges these updated predecessor branches into the + // dependent's existing branch before the agent runs. + orchestration_merge_branches: JSON.stringify(step.mergeBranches), + }, + idempotencyKey, + }, + idempotencyKey, + ); + logger.info('Restack cascade: created restack task for dependent', { + orchestration_id: child.orchestration_id, + sub_issue_id: child.sub_issue_id, + pr_number: prNumber, + status_code: result.statusCode, + }); + // 201 = newly created, 200 = idempotent replay (task already existed from a + // prior delivery of this same source event). Only 201 should surface a + // user-facing comment; 200 means we already did. Other codes = not created. + if (result.statusCode === 201) return 'created'; + if (result.statusCode === 200) return 'exists'; + return 'failed'; + } catch (err) { + logger.error('Restack cascade: createTaskCore threw for dependent', { + orchestration_id: child.orchestration_id, + sub_issue_id: child.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + return 'failed'; + } +} + +/** + * Read a dependent's PR number from its TaskRecord. Prefers numeric + * ``pr_number``; orchestration child tasks commonly persist only ``pr_url`` + * (``.../pull/N``) with ``pr_number`` null — fall back to parsing it. + */ +/** The dependent's PR URL (for a clickable reply link). Null when absent. */ +async function resolvePrUrl(taskId?: string): Promise { + if (!taskId) return null; + try { + const res = await ddb.send(new GetCommand({ + TableName: TASK_TABLE, Key: { task_id: taskId }, ProjectionExpression: 'pr_url', + })); + return typeof res.Item?.pr_url === 'string' ? res.Item.pr_url : null; + } catch { + return null; + } +} + +async function resolvePrNumber(taskId?: string): Promise { + if (!taskId) return null; + try { + const res = await ddb.send(new GetCommand({ TableName: TASK_TABLE, Key: { task_id: taskId } })); + const pr = res.Item?.pr_number; + if (typeof pr === 'number') return pr; + const url = res.Item?.pr_url; + if (typeof url === 'string') { + const m = url.match(/\/pull\/(\d+)\b/); + if (m) return Number(m[1]); + } + return null; + } catch (err) { + logger.warn('Restack cascade: failed to read dependent TaskRecord for PR number', { + task_id: taskId, + error: err instanceof Error ? err.message : String(err), + }); + return null; + } +} + +/** + * Lambda entry point — TaskTable stream handler. + * + * Processes records sequentially; a failure on one record throws so the + * stream retries the batch (idempotent replay is safe). Non-terminal / + * non-orchestration records are skipped cheaply. + */ +export async function handler(event: DynamoDBStreamEvent): Promise { + let processed = 0; + // Per-record isolation. A thrown record is reported as a + // batch item failure (by its stream sequence number) so ONLY it retries, + // instead of failing the whole batch and re-driving its healthy siblings. + const batchItemFailures: { itemIdentifier: string }[] = []; + for (const record of event.Records) { + const seq = record.dynamodb?.SequenceNumber; + try { + const evt = parseTerminalTaskRecord(record); + if (!evt) continue; + // Restack cascade: an iteration/restack task on a node X (NOT a child-row task) + // re-stacks X's direct dependents. Routed here, not through child gating. + if (evt.cascadeSubIssueId) { + await cascadeRestack(evt); + } else { + await reconcileTerminalChild(evt); + } + processed += 1; + } catch (err) { + logger.error('Orchestration reconciler record failed — reporting for isolated retry', { + sequence_number: seq, + event_name: record.eventName, + error: err instanceof Error ? err.message : String(err), + }); + // Without a sequence number we can't report the item individually; rethrow + // so the batch fails rather than silently dropping a real error. + if (!seq) throw err; + batchItemFailures.push({ itemIdentifier: seq }); + } + } + logger.info('Orchestration reconciler batch processed', { + records: event.Records.length, + reconciled: processed, + failed: batchItemFailures.length, + }); + return { batchItemFailures }; +} + +function isConditionalCheckFailed(err: unknown): boolean { + return ( + typeof err === 'object' + && err !== null + && 'name' in err + && (err as { name?: string }).name === 'ConditionalCheckFailedException' + ); +} diff --git a/cdk/src/handlers/reconcile-stranded-orchestrations.ts b/cdk/src/handlers/reconcile-stranded-orchestrations.ts new file mode 100644 index 000000000..a9a25ba5a --- /dev/null +++ b/cdk/src/handlers/reconcile-stranded-orchestrations.ts @@ -0,0 +1,358 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Scheduled backstop for Linear orchestration. + * + * The live reconciler (``orchestration-reconciler``) releases + * dependency-unblocked children by reacting to TaskTable-stream terminal + * events. If the reconciler is unavailable when a child reaches terminal + * state (deploy window, throttle, OOM, a poison-record batch parked in + * the DLQ), **that stream event is lost and never reprocessed** — the + * dependent children never get released and the orchestration stalls + * forever with no recovery. + * + * Observed in practice: a child reached COMPLETED during a + * reconciler OOM window; after the fix deployed, the completion event was + * gone, so its dependent stayed ``blocked`` until a manual nudge. + * + * This scheduled sweep is the recovery path. It also closes the + * crash-after-flip hole in the release path: a child stuck + * ``released`` whose task is long-terminal, or a ``ready`` child whose + * release never created a task. (The failure modes this covers are + * analysed in ``docs/research/orchestration-reconciler-correctness.md``.) + * + * For each active orchestration it re-derives the gating truth from + * persisted state and: + * - releases any ``blocked``/``ready`` child whose predecessors are all + * ``succeeded`` (lost release-event recovery), and + * - re-evaluates children whose own task already reached terminal but + * whose row never advanced (lost terminal-event recovery), advancing + * the row + cascading skips/releases accordingly. + * + * Idempotent: ``releaseChild`` is idempotency-keyed and the row flips are + * conditional, so re-running the sweep (or racing the live reconciler) is + * safe. + */ + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { + DynamoDBDocumentClient, + ScanCommand, + GetCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; +import { refreshPanelAndSettle } from './orchestration-reconciler'; +import { createTaskCore } from './shared/create-task-core'; +import { logger } from './shared/logger'; +import { readConcurrencyBudget, releaseReadyChildren } from './shared/orchestration-release'; +import { + loadOrchestration, + ORCHESTRATION_META_SK, + type OrchestrationChildRow, +} from './shared/orchestration-store'; +import { TaskStatus, type TaskStatusType } from '../constructs/task-status'; + +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({})); +const ORCHESTRATION_TABLE = process.env.ORCHESTRATION_TABLE_NAME!; +const TASK_TABLE = process.env.TASK_TABLE_NAME!; +// Throttle the sweep's releases to the user's free concurrency budget +// too (it is the drain path for children left ``ready`` by the live +// reconciler's throttle). Unset → release-all (back-compat; admission gates). +const USER_CONCURRENCY_TABLE = process.env.USER_CONCURRENCY_TABLE_NAME; +const MAX_CONCURRENT = Number(process.env.MAX_CONCURRENT_TASKS_PER_USER ?? '10'); + +/** Terminal child-statuses (orchestration-local). */ +const TERMINAL_CHILD = new Set(['succeeded', 'failed', 'skipped']); + +/** A task is success for gating iff COMPLETED with build not-failed. */ +function taskIsSuccess(rec: Record | undefined): boolean { + return rec?.status === TaskStatus.COMPLETED && rec?.build_passed !== false; +} +function taskIsTerminal(status: TaskStatusType | undefined): boolean { + return status === TaskStatus.COMPLETED || status === TaskStatus.FAILED + || status === TaskStatus.CANCELLED || status === TaskStatus.TIMED_OUT; +} + +/** Scan the table for parent-meta rows → one per orchestration. */ +async function findOrchestrationIds(): Promise { + const ids: string[] = []; + let lastKey: Record | undefined; + do { + const resp = await ddb.send(new ScanCommand({ + TableName: ORCHESTRATION_TABLE, + FilterExpression: 'sub_issue_id = :meta', + ExpressionAttributeValues: { ':meta': ORCHESTRATION_META_SK }, + ProjectionExpression: 'orchestration_id', + ExclusiveStartKey: lastKey as Record | undefined, + })); + for (const item of resp.Items ?? []) { + if (item.orchestration_id) ids.push(item.orchestration_id as string); + } + lastKey = resp.LastEvaluatedKey as Record | undefined; + } while (lastKey); + return ids; +} + +/** Fetch a released child's task record (status + build_passed). */ +async function getTaskRecord(taskId: string): Promise | undefined> { + const res = await ddb.send(new GetCommand({ TableName: TASK_TABLE, Key: { task_id: taskId } })); + return res.Item; +} + +/** + * Reconcile one orchestration from persisted truth. Returns the number of + * children released by this pass (for logging/metrics). + */ +async function reconcileOrchestration(orchestrationId: string): Promise { + const snap = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + if (!snap) return 0; + + const now = new Date().toISOString(); + + // Already fully terminal: no children to release/recover — BUT the parent may + // never have been settled (the last terminal event was lost, so the DDB rows + // are all terminal yet the Linear parent is stuck 🔄 In Progress + // forever). Attempt the idempotent settle before returning, instead of the old + // bare skip that guaranteed the epic never closed on the lossy path. + const allTerminal = snap.children.every((c) => TERMINAL_CHILD.has(c.child_status)); + if (allTerminal) { + try { + await refreshPanelAndSettle(orchestrationId, snap.children, snap.meta, now); + } catch (err) { + logger.warn('Sweep settle (already-terminal epic) failed (will retry next sweep)', { + orchestration_id: orchestrationId, error: err instanceof Error ? err.message : String(err), + }); + } + return 0; + } + + // 1a. Recover a child STUCK in the transient ``releasing`` state (the + // flip-then-create claim): a releaser won the blocked|ready→releasing claim but + // crashed before createTaskCore (or before its rollback). No task exists, + // so reset it to ``ready`` — step 2 re-releases it under a fresh claim. + // Guard on still being ``releasing`` so we don't race a live finalize. + for (const child of snap.children) { + if (child.child_status !== 'releasing') continue; + try { + await ddb.send(new UpdateCommand({ + TableName: ORCHESTRATION_TABLE, + Key: { orchestration_id: orchestrationId, sub_issue_id: child.sub_issue_id }, + UpdateExpression: 'SET child_status = :ready, updated_at = :now', + ConditionExpression: 'child_status = :releasing', + ExpressionAttributeValues: { ':ready': 'ready', ':releasing': 'releasing', ':now': now }, + })); + logger.info('Recovered orchestration child stuck in releasing → ready (crashed mid-release)', { + orchestration_id: orchestrationId, sub_issue_id: child.sub_issue_id, + }); + } catch (err) { + // ConditionalCheckFailed = the row moved off 'releasing' (a live finalize + // won) — fine, nothing to recover. Anything else: log and let the next + // sweep retry. + if ((err as { name?: string })?.name !== 'ConditionalCheckFailedException') { + logger.warn('Failed to recover releasing child (will retry next sweep)', { + orchestration_id: orchestrationId, + sub_issue_id: child.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } + + // 1b. Recover LOST TERMINAL events: a ``released`` child whose task has + // already reached terminal but whose row never advanced. Advance the + // row to succeeded/failed so step 2 can gate dependents correctly. + for (const child of snap.children) { + if (child.child_status !== 'released' || !child.child_task_id) continue; + const rec = await getTaskRecord(child.child_task_id); + if (!taskIsTerminal(rec?.status as TaskStatusType | undefined)) continue; // still running + const newStatus = taskIsSuccess(rec) ? 'succeeded' : 'failed'; + await advanceChildStatus(orchestrationId, child.sub_issue_id, newStatus, now); + } + + // 2. Re-load (statuses may have advanced) and release any blocked/ready + // child whose predecessors are all succeeded, plus skip children with + // a failed predecessor. Derive everything from the fresh persisted + // state — the same truth the live reconciler uses. + const fresh = await loadOrchestration(ddb, ORCHESTRATION_TABLE, orchestrationId); + if (!fresh) return 0; + + const statusOf = new Map(fresh.children.map((c) => [c.sub_issue_id, c.child_status])); + + // Cascade skips: any child with a failed/skipped predecessor → skipped. + let changed = true; + while (changed) { + changed = false; + for (const c of fresh.children) { + if (statusOf.get(c.sub_issue_id) !== 'blocked' && statusOf.get(c.sub_issue_id) !== 'ready') continue; + const deadDep = c.depends_on.some((d) => { + const s = statusOf.get(d); + return s === 'failed' || s === 'skipped'; + }); + if (deadDep) { + // Only skip from blocked/ready. A concurrent recovery could + // have re-released this child (blocked → releasing/released) between our + // snapshot and this write; the source-state guard makes that write a + // no-op instead of clobbering a running child into terminal 'skipped'. + await advanceChildStatus(orchestrationId, c.sub_issue_id, 'skipped', now, ['blocked', 'ready']); + statusOf.set(c.sub_issue_id, 'skipped'); + changed = true; + } + } + } + + // Releasable: children whose deps are ALL succeeded and that have NOT yet + // started a task. Includes: + // - ``blocked`` children (lost release-event recovery), and + // - ``ready`` children with no ``child_task_id`` — left un-started by the + // live reconciler's concurrency throttle (or a prior create_failed). + // A ``ready`` child that already has a task was genuinely released; re- + // releasing is idempotent, but we skip it to keep the budget for new work. + const releasableRows: OrchestrationChildRow[] = fresh.children + .filter((c) => { + const s = statusOf.get(c.sub_issue_id); + const depsReady = c.depends_on.every((d) => statusOf.get(d) === 'succeeded'); + if (!depsReady) return false; + if (s === 'blocked') return true; + // `ready`, deliberately id-agnostic: a ready child left un-released by the + // concurrency throttle OR reset-to-ready by an epic retry (which KEEPS its + // prior child_task_id to salt the idempotency key). The old + // `!child_task_id` gate stranded retried children forever. Re-releasing is + // safe: the salt spawns a fresh task and the flip-then-create claim is the + // race guard. + if (s === 'ready') return true; + return false; + }) + .map((c) => ({ ...c, child_status: 'ready' as const })); + + if (releasableRows.length === 0) { + // Nothing left to release. If EVERY child is now terminal + // but the parent was never settled — because the last child's terminal stream + // event was lost (the exact failure this sweep exists to recover) — the epic + // is 'done' in DynamoDB yet stuck showing 🔄 In Progress in Linear forever. + // The panel/rollup/settle side-effects lived ONLY on the (lossy) live-event + // path. Call the shared settle here so the sweep closes the loop: post the + // rollup + transition the parent. refreshPanelAndSettle is idempotent (body + // edit is a no-op if unchanged; the parent-state mirror is claimed once), so + // racing the live reconciler is safe. + const freshAllTerminal = fresh.children.every((c) => TERMINAL_CHILD.has(c.child_status)); + if (freshAllTerminal) { + try { + await refreshPanelAndSettle(orchestrationId, fresh.children, fresh.meta, now); + logger.warn('Stranded orchestration settled by sweep — parent was never settled (lost terminal event)', { + orchestration_id: orchestrationId, parent_linear_issue_id: fresh.meta.parent_issue_ref, + }); + } catch (err) { + logger.warn('Sweep settle failed (will retry next sweep)', { + orchestration_id: orchestrationId, error: err instanceof Error ? err.message : String(err), + }); + } + } + return 0; + } + + // Throttle the sweep's releases to the free budget too. + const budget = USER_CONCURRENCY_TABLE + ? await readConcurrencyBudget(ddb, USER_CONCURRENCY_TABLE, fresh.meta.release_context.platform_user_id, MAX_CONCURRENT) + : undefined; + const results = await releaseReadyChildren( + ddb, ORCHESTRATION_TABLE, releasableRows, fresh.meta.release_context, createTaskCore, now, + // Full child set for predecessor-branch-derived base selection. + fresh.children, + 'main', + budget, + ); + const released = results.filter((r) => r.kind === 'released').length; + if (released > 0) { + logger.warn('Stranded orchestration recovered — released children the live reconciler missed', { + orchestration_id: orchestrationId, + released, + candidates: releasableRows.length, + }); + } + return released; +} + +/** + * Conditionally advance a child row's status (no-op if already there). + * + * ``fromStates``: when provided, the write additionally + * asserts the CURRENT status is one of them — so a transition can't clobber a + * child that has since moved to a state outside the allowed source set. The skip + * cascade passes ``['blocked','ready']`` so a live ``released``/``releasing`` + * child (a recovery re-released it while this sweep raced) is NEVER stamped the + * terminal ``skipped`` — which would let the epic claim its once-only rollup + * while that child's work is still running. Recovery advances (released → + * succeeded/failed) pass no ``fromStates`` (any non-target source is fine). + */ +async function advanceChildStatus( + orchestrationId: string, + subIssueId: string, + status: string, + now: string, + fromStates?: readonly string[], +): Promise { + const values: Record = { ':s': status, ':now': now }; + let condition = 'child_status <> :s'; + if (fromStates && fromStates.length > 0) { + const names = fromStates.map((st, i) => { + values[`:from${i}`] = st; + return `:from${i}`; + }); + condition = `child_status <> :s AND child_status IN (${names.join(', ')})`; + } + try { + await ddb.send(new UpdateCommand({ + TableName: ORCHESTRATION_TABLE, + Key: { orchestration_id: orchestrationId, sub_issue_id: subIssueId }, + UpdateExpression: 'SET child_status = :s, updated_at = :now', + ConditionExpression: condition, + ExpressionAttributeValues: values, + })); + } catch (err) { + if ((err as { name?: string })?.name === 'ConditionalCheckFailedException') return; + throw err; + } +} + +/** + * Scheduled entry point. Sweeps every active orchestration. A failure on + * one orchestration is logged and does not abort the rest. + */ +export async function handler(): Promise { + const ids = await findOrchestrationIds(); + let totalReleased = 0; + let swept = 0; + for (const id of ids) { + try { + totalReleased += await reconcileOrchestration(id); + swept += 1; + } catch (err) { + logger.error('Stranded-orchestration sweep failed for one orchestration (continuing)', { + orchestration_id: id, + error: err instanceof Error ? err.message : String(err), + }); + } + } + logger.info('Stranded-orchestration sweep complete', { + orchestrations_swept: swept, + orchestrations_found: ids.length, + children_released: totalReleased, + }); +} diff --git a/cdk/src/handlers/shared/iteration-cost.ts b/cdk/src/handlers/shared/iteration-cost.ts new file mode 100644 index 000000000..56502bb28 --- /dev/null +++ b/cdk/src/handlers/shared/iteration-cost.ts @@ -0,0 +1,144 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { type DynamoDBDocumentClient, QueryCommand, GetCommand } from '@aws-sdk/lib-dynamodb'; +import { logger } from './logger'; +import { TaskTable } from '../../constructs/task-table'; + +/** + * Cap on the number of task rows summed for one issue. + * + * A bound is needed because each row costs a GetItem, so an issue with a + * pathological number of iterations would otherwise fan out unboundedly inside a + * single Lambda invocation. It is deliberately far above any real iteration count + * (dozens at most) so it is a runaway guard, not a routine limit — and when it + * does bite, {@link sumIterationCostForIssue} says so rather than quietly + * reporting a smaller number as if it were the total. + */ +const MAX_COST_TASKS_PER_ISSUE = 500; + +/** Outcome of a running-total cost sum. */ +export interface IterationCostTotal { + /** Summed cost, or null when nothing is known (no rows, all unreadable). */ + readonly total: number | null; + /** + * True when the sum is known to be INCOMPLETE — the row cap was hit, or a read + * failed partway. Callers that show this to a user should mark it as a partial + * figure; a number presented as a total when it is not is worse than no number. + */ + readonly partial: boolean; +} + +/** + * Sum ``cost_usd`` across every task recorded against one Linear issue — the + * running total shown on an iteration's settle reply. + * + * Single implementation shared by the reconciler and the fan-out dispatcher. It + * previously existed as two near-copies that had already drifted: one parsed a + * string ``cost_usd`` and added it without a finite check (so a stringified cost + * poisoned the whole total to NaN), the other guarded correctly. Cost accounting + * that disagrees with itself depending on which handler ran is worse than either + * behaviour on its own. + * + * PAGINATES. A DynamoDB Query returns at most 1 MB per page, and the previous + * versions summed a single page as though it were everything — so past the page + * boundary the user was shown a total that was silently short. Truncation here + * surfaces as a wrong number rather than an error, which is exactly the failure + * mode worth spending a loop to avoid. + * + * ``thisCost`` is added explicitly because the terminal task's GSI projection may + * not have propagated yet; it is deduped by ``task_id`` so it cannot be counted + * twice. + * + * Best-effort: never throws. On a read failure it returns what it has, flagged + * ``partial``. + */ +export async function sumIterationCostForIssue(args: { + ddb: DynamoDBDocumentClient; + taskTableName: string; + linearIssueId: string; + thisTaskId: string; + thisCost?: number; + logLabel?: string; +}): Promise { + const { ddb, taskTableName, linearIssueId, thisTaskId, thisCost, logLabel } = args; + const base = typeof thisCost === 'number' && Number.isFinite(thisCost) ? thisCost : 0; + const parseCost = (v: unknown): number => + typeof v === 'number' ? v : (typeof v === 'string' ? Number(v) : NaN); + + let total = 0; + let sawThis = false; + let partial = false; + try { + const ids: string[] = []; + let startKey: Record | undefined; + do { + const listed = await ddb.send(new QueryCommand({ + TableName: taskTableName, + IndexName: TaskTable.LINEAR_ISSUE_INDEX, + KeyConditionExpression: 'linear_issue_id = :iid', + ProjectionExpression: 'task_id', + ExpressionAttributeValues: { ':iid': linearIssueId }, + ...(startKey && { ExclusiveStartKey: startKey }), + })); + for (const item of (listed.Items ?? []) as Array<{ task_id?: string }>) { + if (item.task_id) ids.push(item.task_id); + } + startKey = listed.LastEvaluatedKey as Record | undefined; + if (ids.length >= MAX_COST_TASKS_PER_ISSUE) { + // Trim to the cap, don't just stop paginating. The cap has to bound the + // GetItem fan-out below, and one Query page can hold far more than the cap + // (the projection is task_id alone, so ~1 MB is a great many rows) — so + // breaking out without trimming would still fan out over the whole page. + ids.length = MAX_COST_TASKS_PER_ISSUE; + // Say so rather than trimming in silence: a capped sum is not the total. + logger.warn('Iteration cost sum hit the per-issue row cap — reporting a PARTIAL total', { + linear_issue_id: linearIssueId, cap: MAX_COST_TASKS_PER_ISSUE, ...(logLabel && { source: logLabel }), + }); + partial = true; + break; + } + } while (startKey); + + // The GSI lists task ids but does not project cost_usd (a GSI projection + // cannot be changed in place — see task-table.ts), so read each cost. + for (const taskId of ids) { + if (taskId === thisTaskId) { + sawThis = true; + total += base; + continue; + } + const got = await ddb.send(new GetCommand({ + TableName: taskTableName, Key: { task_id: taskId }, ProjectionExpression: 'cost_usd', + })); + const c = parseCost(got.Item?.cost_usd); + if (Number.isFinite(c)) total += c; + } + } catch (err) { + logger.warn('Iteration running-total cost query failed — reporting a PARTIAL total', { + task_id: thisTaskId, + error: err instanceof Error ? err.message : String(err), + ...(logLabel && { source: logLabel }), + }); + partial = true; + } + + if (!sawThis) total += base; + return { total: total > 0 ? total : null, partial }; +} diff --git a/cdk/src/handlers/shared/orchestration-discovery.ts b/cdk/src/handlers/shared/orchestration-discovery.ts new file mode 100644 index 000000000..a0f571edf --- /dev/null +++ b/cdk/src/handlers/shared/orchestration-discovery.ts @@ -0,0 +1,245 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Orchestration discovery composer. + * + * Ties together the three discovery primitives in one decision function the + * webhook processor calls when a parent issue is labeled: + * + * fetchSubIssueGraph → validateDag → seedOrchestration + * + * and returns a single discriminated outcome the caller acts on: + * + * - ``single_task`` — the issue has no sub-issues; the caller should + * fall through to today's one-issue→one-task path (NOT an error). + * - ``seeded`` — a valid DAG was persisted; the reconciler + * will release children. Carries the orchestration id + initial + * ready (root) set so the caller or the reconciler can start them. + * - ``rejected`` — the graph is invalid (cycle / dangling / dup). + * Carries a user-facing message for the terminal Linear comment; + * nothing is persisted. + * - ``error`` — transient failure reaching Linear; the caller + * surfaces a retryable message and does NOT fall back to a single + * task (that would silently drop the epic structure). + * + * The DAG validation + persistence are pure/injected, so this composer + * is fully unit-testable with a mock fetch + mock ddb. + */ + +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { logger } from './logger'; +import { validateDag } from './orchestration-dag'; +import { type OrchestrationGraphSource } from './orchestration-graph-source'; +import { withIntegrationNode } from './orchestration-integration-node'; +import { deriveOrchestrationId, extendOrchestration, OrchestrationIdCollisionError, seedOrchestration, type OrchestrationReleaseContext } from './orchestration-store'; + +export interface DiscoverOrchestrationParams { + readonly ddb: DynamoDBDocumentClient; + readonly tableName: string; + readonly parentIssueRef: string; + readonly credentialsRef: string; + readonly repo: string; + /** ISO timestamp injected for testability. */ + readonly now: string; + /** Optional TTL epoch seconds for the persisted rows. */ + readonly ttl?: number; + /** Release context stamped on the meta row for the reconciler. */ + readonly releaseContext: OrchestrationReleaseContext; + /** + * The producer of the orchestration DAG — REQUIRED, so the caller always + * states where the graph comes from. It used to be optional and fell back to + * reading Linear, which meant a caller on any other surface silently made a + * Linear API call for a graph that was never there. Pass a native source for a + * surface with its own dependency model, or ``declarativeGraphSource`` when the + * nodes are already in hand (a CLI/API request, or a planner's output). The + * validate→seed→reconcile→rollup pipeline downstream is identical either way. + */ + readonly graphSource: OrchestrationGraphSource; +} + +export type DiscoverOrchestrationResult = + | { readonly kind: 'single_task'; readonly parentIssueRef: string } + | { + readonly kind: 'seeded'; + readonly orchestrationId: string; + readonly childCount: number; + readonly rootSubIssueIds: readonly string[]; + readonly alreadyExisted: boolean; + } + | { + // An already-seeded orchestration that was EXTENDED with sub-issues + // added to the epic after the first seed (orchestration-extend). Carries + // the new node ids + which are immediately releasable. + readonly kind: 'extended'; + readonly orchestrationId: string; + readonly addedSubIssueIds: readonly string[]; + readonly releasableSubIssueIds: readonly string[]; + } + | { readonly kind: 'rejected'; readonly reason: string; readonly message: string } + | { readonly kind: 'error'; readonly message: string }; + +/** + * Discover, validate, and persist a parent issue's sub-issue DAG. + * Never throws — all failure modes are returned as discriminated + * results so the webhook processor can map each to the right + * user-facing behaviour. + */ +export async function discoverOrchestration( + params: DiscoverOrchestrationParams, +): Promise { + const { ddb, tableName, parentIssueRef, credentialsRef, repo, now, ttl, releaseContext, graphSource } = params; + + // ── 1. Produce the orchestration graph ─────────────────────────── + // The caller states the source; this composer never reaches for a surface of + // its own. The downstream pipeline is identical regardless of where the graph + // came from. + const fetched = await graphSource(); + if (fetched.kind === 'error') { + return { kind: 'error', message: fetched.message }; + } + if (fetched.kind === 'no_children') { + logger.info('No orchestration graph — falling back to single task', { + parent_issue_ref: parentIssueRef, + }); + return { kind: 'single_task', parentIssueRef }; + } + + // ── 2. Validate the DAG (cycle / dangling / duplicate rejection) ─ + const validation = validateDag(fetched.children); + if (!validation.ok) { + logger.warn('Orchestration DAG rejected', { + parent_issue_ref: parentIssueRef, + reason: validation.reason, + offending_ids: validation.offendingIds, + }); + return { kind: 'rejected', reason: validation.reason, message: validation.message }; + } + + // ── 2b. Auto-integration node for fan-out. If the validated DAG has + // >1 leaf, append a synthetic node depending on all leaves so a pure + // fan-out still produces ONE combined result (the node is a diamond + // fan-in, reusing the existing predecessor merge). No-op for linear chains / explicit + // diamonds (≤1 leaf). The orchestration id is derived deterministically + // from the parent issue, so we can name the synthetic node before seeding. + const orchestrationId = deriveOrchestrationId(parentIssueRef); + const augmented = withIntegrationNode(fetched.children, orchestrationId); + let childrenToSeed = augmented.nodes; + if (augmented.added) { + // Re-validate defensively — appending a fan-in over leaves cannot + // introduce a cycle/dangle/dup, but seeding an invalid graph would be + // worse than skipping the synthetic node, so fail-safe to the + // un-augmented graph if it ever does. + const reValidation = validateDag(childrenToSeed); + if (!reValidation.ok) { + logger.error('Integration node produced an invalid DAG — seeding without it', { + parent_issue_ref: parentIssueRef, + reason: reValidation.reason, + }); + childrenToSeed = fetched.children; + } else { + logger.info('Orchestration fan-out detected — added integration node', { + parent_issue_ref: parentIssueRef, + orchestration_id: orchestrationId, + // the synthetic node is last; its predecessors are the leaves it merges + leaf_count: childrenToSeed[childrenToSeed.length - 1].depends_on.length, + }); + } + } + + // ── 3. Persist (idempotent on replay) ──────────────────────────── + let seedResult; + try { + seedResult = await seedOrchestration({ + ddb, + tableName, + parentIssueRef, + credentialsRef, + repo, + children: childrenToSeed, + now, + releaseContext, + ...(ttl !== undefined && { ttl }), + }); + } catch (err) { + logger.error('Failed to persist orchestration graph', { + parent_issue_ref: parentIssueRef, + error: err instanceof Error ? err.message : String(err), + }); + // A collision will recur on every attempt, so don't invite a re-trigger the + // way a transient write failure does — that would loop the user through + // something that cannot succeed until the ref scheme changes. + if (err instanceof OrchestrationIdCollisionError) { + return { + kind: 'error', + message: 'This issue maps to an orchestration that already belongs to a different issue or workspace, ' + + 'so I stopped rather than acting on the wrong graph. This needs an operator to look at it.', + }; + } + return { kind: 'error', message: 'Could not persist the orchestration graph. Please re-apply the trigger.' }; + } + + // ── 3b. Already-seeded → EXTEND with any sub-issues added since the first + // seed (orchestration-extend). seedOrchestration is frozen-at-first-seed, so + // a re-trigger of an existing epic lands here; diff the current graph against + // the persisted children and add genuinely-new nodes. A re-trigger with no + // new nodes is a clean no-op (addedSubIssueIds empty). + if (seedResult.alreadyExisted) { + let extendResult; + try { + extendResult = await extendOrchestration({ + ddb, + tableName, + parentIssueRef, + credentialsRef, + repo, + graph: childrenToSeed, + now, + ...(ttl !== undefined && { ttl }), + }); + } catch (err) { + logger.error('Failed to extend orchestration graph', { + parent_issue_ref: parentIssueRef, + error: err instanceof Error ? err.message : String(err), + }); + return { kind: 'error', message: 'Could not extend the orchestration graph. Please re-apply the trigger.' }; + } + if (extendResult.rejected) { + return { kind: 'rejected', reason: extendResult.rejected.reason, message: extendResult.rejected.message }; + } + return { + kind: 'extended', + orchestrationId: extendResult.orchestrationId, + addedSubIssueIds: extendResult.addedSubIssueIds, + releasableSubIssueIds: extendResult.releasableSubIssueIds, + }; + } + + // Roots = layer 0 of the validated topological layering. The + // reconciler releases these first. + const rootSubIssueIds = validation.layers[0] ?? []; + + return { + kind: 'seeded', + orchestrationId: seedResult.orchestrationId, + childCount: childrenToSeed.length, + rootSubIssueIds, + alreadyExisted: seedResult.alreadyExisted, + }; +} diff --git a/cdk/src/handlers/shared/orchestration-parent-comment.ts b/cdk/src/handlers/shared/orchestration-parent-comment.ts new file mode 100644 index 000000000..1754981a9 --- /dev/null +++ b/cdk/src/handlers/shared/orchestration-parent-comment.ts @@ -0,0 +1,368 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Pure logic for routing an ``@bgagent`` comment left on the PARENT epic to the + * sub-issue it's about. + * + * Background: the maturing epic panel lives on the parent epic, so a reviewer's + * natural instinct is to comment there ("@bgagent for the footer, change X"). + * But the parent epic has no PR of its own — only its sub-issues do — so the + * comment-trigger path can't iterate "the parent". Previously such a comment + * fell through to the standalone path, found no task for the parent issue, and + * was SILENTLY DROPPED (observed in practice). This module decides, from the + * instruction text + the orchestration's sub-issue rows, WHICH sub-issue the + * comment targets so the processor can iterate that sub-issue's PR. + * + * Pure (no I/O) so the matching is unit-tested in isolation; the processor does + * the Linear/DDB work (resolve PR, spawn the iteration task, ack). + */ + +import { KNOWN_EPIC_COMMANDS } from './orchestration-comment-trigger'; +import { isIntegrationNode } from './orchestration-integration-node'; + +/** Minimal view of a sub-issue row this matcher needs. */ +export interface ParentCommentNode { + readonly sub_issue_id: string; + /** + * The human-facing id a commenter would type (``ENG-42``). Read through + * {@link nodeDisplayId} rather than directly: persisted rows carry it under + * either this neutral name or the older surface-specific one depending on when + * they were written, and reading only one of the two silently loses + * name-the-node routing for the rows that use the other. + */ + readonly display_id?: string; + /** Legacy name for {@link display_id}, still present on rows written earlier. */ + readonly linear_identifier?: string; + readonly title?: string; + /** + * Sub-issue description. NOT used for act-routing (too low-precision to + * auto-iterate on — a long description shares words with many comments), but + * it IS scored for the "did you mean …?" SUGGESTION so a comment like + * "change the header color to yellow instead of blue" surfaces the header + * node (whose description mentions blue) for a one-tap confirm, rather than a + * generic "couldn't tell". Optional — matching degrades gracefully when absent. + */ + readonly description?: string; + /** Only a STARTED child (has a task) can be iterated; the matcher reports it but the caller gates on a PR. */ + readonly child_task_id?: string; +} + +export interface ParentNodeMatch { + /** Sub-issues the instruction plausibly targets (excludes the synthetic integration node unless named). */ + readonly matches: readonly ParentCommentNode[]; + /** + * Why the caller can't act on exactly one node: + * - 'none' — no node referenced (generic comment like "@bgagent looks good") + * - 'ambiguous' — the text matched more than one node + * - null — exactly one match (caller iterates it) + */ + readonly reason: 'none' | 'ambiguous' | null; +} + +/** Lowercase, collapse whitespace, strip punctuation that breaks word matching. */ +function normalize(s: string): string { + return s.toLowerCase().replace(/[^a-z0-9\s-]/g, ' ').replace(/\s+/g, ' ').trim(); +} + +/** + * Title "noise" words that carry no routing signal — matching on them would + * make every comment hit every node. We only match a node by its title when a + * SIGNIFICANT (non-noise) word from the title appears in the instruction. + */ +const TITLE_NOISE = new Set([ + 'add', 'a', 'an', 'the', 'to', 'of', 'for', 'and', 'or', 'with', 'new', + 'page', 'section', 'site', 'wide', 'site-wide', 'update', 'change', 'fix', + 'create', 'make', 'support', 'feature', 'this', 'that', 'can', 'you', 'please', +]); + +/** + * Decide which sub-issue(s) an ``@bgagent`` instruction left on the parent epic + * is about. + * + * Matching, in priority order: + * 1. Linear identifier token (``ENG-42``) — exact, case-insensitive. The + * unambiguous way to target a node; if present it wins outright (a single + * identifier → single match, even if a keyword also matched another node). + * 2. Significant title keyword — a non-noise word from a node's title that + * appears in the instruction (``footer`` → "Add a site-wide footer"). All + * nodes whose title contributes a matched keyword are collected. + * + * The synthetic integration node is excluded from keyword matching (its title + * "Integration — combine sub-issue results" is generic) but CAN be targeted by + * the words "integration"/"combined" or its (nonexistent) identifier — callers + * rarely iterate it, so it only matches on an explicit "integration" mention. + * + * Returns ``reason: null`` only when exactly one node matched. + */ +/** + * The word that explicitly targets the synthetic integration node from a parent + * comment. ``combined`` is accepted as a synonym because that is what the panel + * calls it ("Integration — combined result", "Combined PR", "Combined preview"), + * and a user will reasonably reach for the word they were shown. + * + * Shared by the parser and the disambiguation reply so the phrasing we ADVERTISE + * cannot drift from the phrasing we ACCEPT — the failure mode being a reply that + * tells someone to type a word the parser ignores. + */ +export const INTEGRATION_KEYWORD = 'integration'; +/** Accepted synonym of {@link INTEGRATION_KEYWORD}. */ +export const INTEGRATION_KEYWORD_ALT = 'combined'; + +export function parseParentNodeReference( + instruction: string, + nodes: readonly ParentCommentNode[], +): ParentNodeMatch { + const text = normalize(instruction); + if (!text) return { matches: [], reason: 'none' }; + const tokens = new Set(text.split(' ')); + + // 1) Identifier match wins outright. + const byIdentifier = nodes.filter((n) => { + const id = nodeDisplayId(n); + return id !== undefined && tokens.has(id.toLowerCase()); + }); + if (byIdentifier.length === 1) return { matches: byIdentifier, reason: null }; + if (byIdentifier.length > 1) return { matches: byIdentifier, reason: 'ambiguous' }; + + // 2) Significant-title-keyword match. + const byKeyword = nodes.filter((n) => { + if (isIntegrationNode(n.sub_issue_id)) { + // The keyword IS the match for the synthetic node, not merely a permission + // to then match its title. It previously worked the other way round, so + // `integration` routed only because that word happens to appear in the + // node's title while `combined` — the word the panel actually shows, in + // "combined result" / "Combined PR" / "Combined preview" — matched nothing + // and fell through to a "which sub-issue?" reply. The node has no Linear + // identifier to fall back on, so the keyword is the only handle a user has. + return tokens.has(INTEGRATION_KEYWORD) || tokens.has(INTEGRATION_KEYWORD_ALT); + } + if (!n.title) return false; + const significant = normalize(n.title) + .split(' ') + .filter((w) => w.length > 2 && !TITLE_NOISE.has(w)); + return significant.some((w) => tokens.has(w)); + }); + + if (byKeyword.length === 1) return { matches: byKeyword, reason: null }; + if (byKeyword.length > 1) return { matches: byKeyword, reason: 'ambiguous' }; + return { matches: [], reason: 'none' }; +} + +/** Significant (non-noise, length>2) words of a string, as a Set. */ +function significantWords(s: string | undefined): Set { + if (!s) return new Set(); + return new Set( + normalize(s).split(' ').filter((w) => w.length > 2 && !TITLE_NOISE.has(w)), + ); +} + +/** + * Best-effort "did you mean …?" suggestion for the disambiguation reply, used + * ONLY when {@link parseParentNodeReference} found no confident match. We never + * ACT on this (no silent iteration of a guess) — it's a hint in the reply so + * the human can confirm with one tap. + * + * Scores each real node by overlap with BOTH its title (weighted + * heavily) and its description (weighted lightly). The description tier is what + * lets "change the header color to yellow instead of blue" surface the header + * node — whose title is "...header bar..." (title hit) and/or whose description + * mentions the blue it changes. Title overlap dominates so a description-only + * coincidence can't outrank a real title match. Returns the single best scorer, + * or null when nothing overlaps at all. The synthetic integration node is never + * suggested. + */ +export function suggestClosestNode( + instruction: string, + nodes: readonly ParentCommentNode[], +): ParentCommentNode | null { + const tokens = new Set(normalize(instruction).split(' ').filter(Boolean)); + if (tokens.size === 0) return null; + const TITLE_WEIGHT = 10; + const DESC_WEIGHT = 1; + let best: ParentCommentNode | null = null; + let bestScore = 0; + for (const n of nodes) { + if (isIntegrationNode(n.sub_issue_id)) continue; + const titleHits = [...significantWords(n.title)].filter((w) => tokens.has(w)).length; + const descHits = [...significantWords(n.description)].filter((w) => tokens.has(w)).length; + const score = titleHits * TITLE_WEIGHT + descHits * DESC_WEIGHT; + if (score > bestScore) { + bestScore = score; + best = n; + } + } + return bestScore > 0 ? best : null; +} + +/** + * Heuristic: does the instruction look like a request for NEW work (add a thing + * that isn't one of the existing sub-issues) rather than a change to an existing + * one? When true and nothing matched, the disambiguation reply leads + * with the "create a sub-issue" path instead of the generic "couldn't tell". + * + * Conservative — only fires when the instruction opens with an additive verb + * (add / create / build / introduce / include / also add …). A change verb + * ("change the footer", "make it bigger") is NOT new work. + */ +const NEW_WORK_VERBS = new Set(['add', 'create', 'build', 'introduce', 'include', 'implement']); +/** How many leading words to scan for an additive verb past politeness filler. */ +const NEW_WORK_LEAD_SCAN = 5; +export function looksLikeNewWork(instruction: string): boolean { + const words = normalize(instruction).split(' ').filter(Boolean); + // Scan the first few words for a leading additive verb ("also add ...", + // "can you add ...", "please create ..."), skipping politeness/filler. + const FILLER = new Set(['also', 'can', 'you', 'please', 'could', 'would', 'lets', 'let', 'us', 'we', 'i', 'd', 'like', 'to', 'now', 'maybe']); + for (const w of words.slice(0, NEW_WORK_LEAD_SCAN)) { + if (NEW_WORK_VERBS.has(w)) return true; + if (!FILLER.has(w)) break; // first non-filler word isn't an additive verb → not new work + } + return false; +} + +/** + * The node's human-facing id under either the neutral or the legacy attribute + * name. Rows persisted before and after the rename coexist indefinitely (there + * is no backfill), so every read of this id has to accept both — the neutral name + * wins where a row carries both. + */ +export function nodeDisplayId(n: ParentCommentNode): string | undefined { + return n.display_id ?? n.linear_identifier; +} + +/** + * The integration node's line for the disambiguation reply, or [] when the epic + * has no integration node (a 0–1 leaf graph — the final child IS the combined + * result, so there is nothing separate to target). + * + * Listed but never auto-selected. A vague parent comment must not be routed to + * the combined PR on a guess: the integration branch holds every sibling's work, + * so a misrouted change there is the most expensive kind to unpick. Targeting it + * stays explicit, which is why the line spells out the exact phrasing. + */ +function integrationHint(nodes: readonly ParentCommentNode[]): string[] { + const node = nodes.find((n) => isIntegrationNode(n.sub_issue_id)); + if (!node) return []; + return [ + `- Integration — combined result; target with \`@bgagent ${INTEGRATION_KEYWORD}: \``, + ]; +} + +function nodeLabel(n: ParentCommentNode): string { + const id = nodeDisplayId(n); + if (id) return n.title ? `${id} — ${n.title}` : id; + return n.title ?? n.sub_issue_id; +} + +/** + * Render the "which sub-issue?" threaded reply posted on the parent epic when + * {@link parseParentNodeReference} can't pin exactly one node. NEVER auto-acts + * and NEVER auto-creates an issue (that stays the user's call): it (a) surfaces a + * best-effort "did you mean ?" suggestion when one overlaps, (b) lists the + * real sub-issues + how to target one, and (c) points at the "create a + * sub-issue for NEW work" path. So a parent comment is never silently dropped, + * but new work only ever begins when the human explicitly creates a sub-issue. + * Pure (string only). + * + * @param suggestion best-effort closest node (from {@link suggestClosestNode}), or null + * @param newWork when the instruction looks like a request for NEW + * work (see {@link looksLikeNewWork}), lead with the + * create-a-sub-issue path instead of the generic "couldn't + * tell" — the comment isn't about an existing sub-issue at all. + */ +export function renderParentDisambiguationReply( + reason: 'none' | 'ambiguous', + nodes: readonly ParentCommentNode[], + suggestion?: ParentCommentNode | null, + newWork = false, + /** + * Whether the epic currently has failed/skipped children. + * When true, the reply surfaces the ``retry`` command prominently — so a user + * who typed something we couldn't act on (a typo, a vague ask, anything) is + * ALWAYS shown what they CAN type, rather than us trying to guess their intent. + * The command list is truthful (from {@link KNOWN_EPIC_COMMANDS}) so it stays + * correct as commands are added. + */ + hasFailures = false, +): string { + const real = nodes.filter((n) => !isIntegrationNode(n.sub_issue_id)); + // The "what you can type here" footer — always present so an unrecognised + // comment is never a dead-end. `retry` is only meaningful when something + // failed, so it's listed only then (with the equivalent re-label path, so the + // two ways to retry read consistently). + const commandsFooter = hasFailures + ? [ + '', + // Does NOT bill the label re-apply as the same thing: re-applying resolves to + // one of four outcomes from graph state (add sub-issues / retry / still + // running / already complete), so it is only equivalent in this one state. + `**Commands:** ${KNOWN_EPIC_COMMANDS.map((c) => `\`@bgagent ${c}\``).join(', ')} ` + + 're-runs the failed/skipped sub-issues and keeps the ones that succeeded.', + ] + : []; + + // New-work path leads with the create-a-sub-issue ask (the comment + // is adding something, not changing an existing sub-issue), then lists the + // existing ones for context. Never auto-creates. + if (newWork && reason === 'none') { + return [ + '👋 That looks like **new work** rather than a change to one of the ' + + 'existing sub-issues.', + '', + 'To have me build it, create a new sub-issue under this epic and add the ' + + '`abca` label — I\'ll fold it into the orchestration. (If you actually ' + + 'meant one of the existing sub-issues, name it — e.g. ' + + '`@bgagent ENG-123: `.)', + '', + 'The current sub-issues are:', + '', + ...real.map((n) => `- ${nodeLabel(n)}`), + ...integrationHint(nodes), + ...commandsFooter, + ].join('\n'); + } + + const lead = reason === 'ambiguous' + ? "That could apply to more than one sub-issue, so I didn't want to guess." + : "I couldn't tell which sub-issue that's about."; + const out: string[] = [`👋 ${lead}`, '']; + if (suggestion) { + out.push( + `Did you mean **${nodeLabel(suggestion)}**? If so, reply ` + + `\`@bgagent ${nodeDisplayId(suggestion) ?? 'that one'}: \`.`, + '', + ); + } + out.push( + // "Otherwise" only makes sense against the "Did you mean …?" sentence above, and + // that sentence is conditional — with no suggestion the reply opened with a + // contrast to nothing. + (suggestion ? 'Otherwise, comment' : 'Comment') + + ' on the specific sub-issue, or name it here — e.g. ' + + '`@bgagent ENG-123: `. The sub-issues are:', + '', + ...real.map((n) => `- ${nodeLabel(n)}`), + ...integrationHint(nodes), + '', + "If it's **new work** (not a change to one of these), create a new sub-issue " + + 'under this epic and add the `abca` label — I\'ll fold it into the orchestration.', + ...commandsFooter, + ); + return out.join('\n'); +} diff --git a/cdk/src/handlers/shared/orchestration-reconcile.ts b/cdk/src/handlers/shared/orchestration-reconcile.ts new file mode 100644 index 000000000..eedf96817 --- /dev/null +++ b/cdk/src/handlers/shared/orchestration-reconcile.ts @@ -0,0 +1,374 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Pure gating logic for the orchestration reconciler. Given a child sub-issue + * that just reached a terminal state + * plus the current orchestration rows, decide: + * - the new ``child_status`` for the terminal child, + * - which blocked children become releasable (all predecessors + * succeeded), and + * - which children must be skipped (a predecessor failed → transitive + * dependents never start). + * + * No I/O — the reconciler handler applies the returned plan to + * DynamoDB + ``createTaskCore``. Keeping this pure makes the 8-case + * failure matrix from the design doc directly unit-testable. + */ + +import type { ChildStatus } from './orchestration-store'; + +/** Minimal view of an orchestration child row the gating logic needs. */ +export interface ReconcileChild { + readonly sub_issue_id: string; + readonly depends_on: readonly string[]; + readonly child_status: ChildStatus; +} + +/** The terminal outcome of the child that triggered this reconcile. */ +export interface TerminalOutcome { + readonly sub_issue_id: string; + /** Task terminal status. */ + readonly status: 'COMPLETED' | 'FAILED' | 'CANCELLED' | 'TIMED_OUT'; + /** + * Whether the agent build passed. A child can be ``COMPLETED`` with + * ``build_passed === false`` (PR opened but build failed); we do NOT + * release dependents onto broken code. ``undefined`` is treated as + * "not known to have failed" → still a success for gating (matches + * the TaskRecord field being optional/absent on older records). + */ + readonly build_passed?: boolean; +} + +/** A single child-status mutation the handler must persist. */ +export interface StatusUpdate { + readonly sub_issue_id: string; + readonly child_status: ChildStatus; +} + +export interface ReconcilePlan { + /** ``true`` when the terminal child counts as a success for gating. */ + readonly terminalSucceeded: boolean; + /** Status writes to apply (includes the terminal child itself). */ + readonly statusUpdates: readonly StatusUpdate[]; + /** Sub-issue ids that are now releasable (create child task, mark released). */ + readonly toRelease: readonly string[]; + /** True when every child has reached a terminal orchestration state. */ + readonly orchestrationComplete: boolean; +} + +/** Orchestration-local terminal child statuses. */ +const TERMINAL_CHILD_STATUSES: ReadonlySet = new Set([ + 'succeeded', + 'failed', + 'skipped', +]); + +/** A child counts as "done successfully" for releasing its dependents. */ +function isSuccess(outcome: TerminalOutcome): boolean { + return outcome.status === 'COMPLETED' && outcome.build_passed !== false; +} + +/** + * Compute the reconcile plan for one terminal child. + * + * @param outcome the child that just reached terminal state. + * @param children all rows for the orchestration (including the terminal + * child). ``child_status`` reflects current persisted state. + * + * Gating rules (design §"Failure semantics"): + * - Success: mark the child ``succeeded``. Any ``blocked`` child whose + * predecessors are ALL succeeded (case 2: diamond needs all, not any) + * becomes ``toRelease``. + * - Failure/cancel/timeout, or COMPLETED-with-failed-build (case 1): + * mark the child ``failed``, and transitively mark every dependent + * (direct + indirect) ``skipped`` — they can never start because a + * predecessor will never succeed. + */ +export function computeReconcilePlan( + outcome: TerminalOutcome, + children: readonly ReconcileChild[], +): ReconcilePlan { + const succeeded = isSuccess(outcome); + + // Working copy of statuses so we can reason about "all predecessors + // succeeded" against the post-update world. + const statusOf = new Map( + children.map((c) => [c.sub_issue_id, c.child_status]), + ); + + const updates: StatusUpdate[] = []; + const setStatus = (id: string, s: ChildStatus): void => { + statusOf.set(id, s); + updates.push({ sub_issue_id: id, child_status: s }); + }; + + // 1. The terminal child itself. + setStatus(outcome.sub_issue_id, succeeded ? 'succeeded' : 'failed'); + + const toRelease: string[] = []; + + if (succeeded) { + // 2. Release any blocked child whose predecessors are ALL succeeded. + for (const c of children) { + if (statusOf.get(c.sub_issue_id) !== 'blocked') continue; + const allSucceeded = c.depends_on.every((dep) => statusOf.get(dep) === 'succeeded'); + if (allSucceeded) { + toRelease.push(c.sub_issue_id); + // Mark released so a sibling finishing in the same batch doesn't + // double-release it. + setStatus(c.sub_issue_id, 'released'); + } + } + } else { + // 3. Transitively skip every dependent of the failed child. + // BFS over the reverse-dependency graph. + const dependents = new Map(); + for (const c of children) { + for (const dep of c.depends_on) { + const list = dependents.get(dep) ?? []; + list.push(c.sub_issue_id); + dependents.set(dep, list); + } + } + const queue = [outcome.sub_issue_id]; + const skipped = new Set(); + while (queue.length > 0) { + const cur = queue.shift()!; + for (const dependentId of dependents.get(cur) ?? []) { + if (skipped.has(dependentId)) continue; + const cur_status = statusOf.get(dependentId); + // Only skip children that haven't already started/finished. + // A child already ``released``/``succeeded``/``failed`` is left + // as-is (its own terminal event reconciles it). + if (cur_status === 'blocked' || cur_status === 'ready') { + setStatus(dependentId, 'skipped'); + skipped.add(dependentId); + } + queue.push(dependentId); + } + } + } + + // 4. Is the whole orchestration now terminal? Every child either was + // already terminal or just transitioned to one. ``released`` is NOT + // terminal (the released child's own task is still running). + const orchestrationComplete = children.every((c) => { + const s = statusOf.get(c.sub_issue_id)!; + return TERMINAL_CHILD_STATUSES.has(s); + }); + + return { + terminalSucceeded: succeeded, + statusUpdates: updates, + toRelease, + orchestrationComplete, + }; +} + +export interface RecoveryPlan { + /** Status writes to apply (the un-failed node + any un-skipped dependents). */ + readonly statusUpdates: readonly StatusUpdate[]; + /** Sub-issue ids now releasable (un-skipped because predecessors all succeeded). */ + readonly toRelease: readonly string[]; +} + +/** + * RECOVERY cascade. A human fixed a previously-FAILED sub-issue via a + * comment (``@bgagent …``), its iteration task just succeeded. The forward + * cascade ({@link computeReconcilePlan}) only handles a child reaching terminal + * for the FIRST time; it has no path to *un-fail* a child and re-release the + * dependents that were transitively ``skipped`` when it first failed. This + * computes that recovery: + * + * 1. Flip the recovered node ``failed`` → ``succeeded``. + * 2. Walk its (formerly-skipped) descendants. Any ``skipped`` child whose + * predecessors are now ALL ``succeeded`` becomes releasable (``ready``). + * A descendant with another still-failed/-skipped predecessor stays + * ``skipped`` — recovery is gated the same way the original release was. + * 3. Releasing a child can in turn unblock ITS descendants, so this iterates + * to a fixed point (a chain A→B→C recovered at A re-releases B, then B's + * success later re-releases C via the normal forward cascade — but a + * diamond where the fixed node feeds multiple skipped leaves re-releases + * all whose predecessors are satisfied here in one pass). + * + * Returns empty updates when the node wasn't actually ``failed`` (nothing to + * recover — the normal cascade handles a healthy iteration) so the caller can + * cheaply no-op. Pure (no I/O); the handler persists + releases. + * + * @param recoveredSubIssueId the node whose fix-iteration just succeeded. + * @param children current orchestration rows. + */ +export function computeRecoveryPlan( + recoveredSubIssueId: string, + children: readonly ReconcileChild[], +): RecoveryPlan { + const current = children.find((c) => c.sub_issue_id === recoveredSubIssueId); + // Only meaningful when the node is currently failed. A healthy iteration on a + // succeeded node is the forward cascade's job, not recovery. + if (!current || current.child_status !== 'failed') { + return { statusUpdates: [], toRelease: [] }; + } + + const statusOf = new Map( + children.map((c) => [c.sub_issue_id, c.child_status]), + ); + const updates: StatusUpdate[] = []; + const setStatus = (id: string, s: ChildStatus): void => { + statusOf.set(id, s); + updates.push({ sub_issue_id: id, child_status: s }); + }; + + // 1. Un-fail the recovered node. + setStatus(recoveredSubIssueId, 'succeeded'); + + // 2. Reset EVERY transitively-skipped descendant of the recovered node back to + // 'blocked' — the normal waiting state the forward cascade understands. + // This is the key fix: once a node is 'skipped' the forward cascade + // (computeReconcilePlan) never releases it (it only releases 'blocked' + // nodes), so a deeper node like the integration node would strand skipped + // forever even after its predecessors recover. Putting the whole subtree + // back to 'blocked' lets each layer release normally as predecessors + // succeed: the immediately-ready layer here, deeper layers via the forward + // cascade when their tasks land. BFS over the reverse-dependency graph. + const dependents = new Map(); + for (const c of children) { + for (const dep of c.depends_on) { + const list = dependents.get(dep) ?? []; + list.push(c.sub_issue_id); + dependents.set(dep, list); + } + } + const queue = [recoveredSubIssueId]; + const seen = new Set(); + while (queue.length > 0) { + const cur = queue.shift()!; + for (const depId of dependents.get(cur) ?? []) { + if (seen.has(depId)) continue; + seen.add(depId); + if (statusOf.get(depId) === 'skipped') { + setStatus(depId, 'blocked'); + } + queue.push(depId); + } + } + + // 3. Release any now-'blocked' node whose predecessors are ALL succeeded + // (the immediate layer behind the recovered node). Deeper nodes stay + // 'blocked' and release via the forward cascade as their tasks complete. + // A node with ANOTHER still-failed/-skipped predecessor stays 'blocked' + // (correctly waiting for that one's own recovery) — gated exactly like the + // original release. + const toRelease: string[] = []; + for (const c of children) { + if (statusOf.get(c.sub_issue_id) !== 'blocked') continue; + const allSucceeded = c.depends_on.every((dep) => statusOf.get(dep) === 'succeeded'); + if (allSucceeded) toRelease.push(c.sub_issue_id); + } + + return { statusUpdates: updates, toRelease }; +} + +export interface EpicRetryPlan { + /** Status writes to apply (failed→ready/blocked, skipped→blocked). */ + readonly statusUpdates: readonly StatusUpdate[]; + /** Sub-issue ids now releasable (a reset node with all predecessors succeeded). */ + readonly toRelease: readonly string[]; + /** Count of nodes that were failed before this retry (for the honest reply copy). */ + readonly failedCount: number; + /** Count of nodes that were skipped before this retry. */ + readonly skippedCount: number; + /** Count of nodes left untouched because they already succeeded. */ + readonly succeededCount: number; +} + +/** + * RETRY the whole epic. A human re-applied the trigger label (or + * re-triggered) on a parent whose orchestration is already TERMINAL: some + * children ``failed`` (and their dependents transitively ``skipped``). The + * seed/extend paths don't re-run terminal children — the seed path only releases + * on first-seed, extend only releases genuinely-NEW nodes — so a bare re-trigger + * of a finished-with-failures epic previously re-ran nothing while the note + * claimed it was "running the existing sub-issue graph" (the misleading copy the + * user hit). This makes a re-trigger a real "retry the failed parts": + * + * 1. Every ``failed`` node → ``ready`` if all its predecessors are ``succeeded``, + * else ``blocked`` (its own failed/skipped predecessor is being retried too, + * and the forward cascade releases it once that predecessor re-succeeds). + * 2. Every ``skipped`` node → ``blocked`` (it never ran; put it back in the + * waiting state the forward cascade understands, exactly like recovery). + * 3. ``succeeded`` nodes are LEFT ALONE — we never re-run work that landed. + * 4. Release every now-``ready`` node whose predecessors are ALL ``succeeded`` + * (the immediate layer); deeper layers release via the forward cascade as + * their retried predecessors re-succeed. + * + * Returns an all-zero-count / empty plan when NOTHING is failed or skipped (a + * healthy or still-running epic) so the caller can distinguish "retried N" from + * "nothing to retry" and post honest copy. Pure (no I/O); the handler persists + + * releases + resets the reconcile-complete marker. Mirrors {@link computeRecoveryPlan} + * but keyed on the whole graph rather than one recovered node. + */ +export function computeEpicRetryPlan( + children: readonly ReconcileChild[], +): EpicRetryPlan { + const statusOf = new Map( + children.map((c) => [c.sub_issue_id, c.child_status]), + ); + const failedCount = children.filter((c) => c.child_status === 'failed').length; + const skippedCount = children.filter((c) => c.child_status === 'skipped').length; + const succeededCount = children.filter((c) => c.child_status === 'succeeded').length; + + // Nothing to retry — no failed/skipped nodes. Empty plan; the caller reports + // honestly (already running, or already all-succeeded) instead of re-releasing. + if (failedCount === 0 && skippedCount === 0) { + return { statusUpdates: [], toRelease: [], failedCount, skippedCount, succeededCount }; + } + + const updates: StatusUpdate[] = []; + const setStatus = (id: string, s: ChildStatus): void => { + statusOf.set(id, s); + updates.push({ sub_issue_id: id, child_status: s }); + }; + + // 1 + 2. Reset failed → ready/blocked (by whether preds are already succeeded) + // and skipped → blocked. We compute failed→ready against the CURRENT + // (pre-reset) statuses first so a failed node whose preds all succeeded + // goes straight to ready; a failed node behind another failed/skipped + // node goes blocked and waits for the forward cascade. + for (const c of children) { + if (c.child_status === 'failed') { + const allDepsSucceeded = c.depends_on.every((dep) => statusOf.get(dep) === 'succeeded'); + setStatus(c.sub_issue_id, allDepsSucceeded ? 'ready' : 'blocked'); + } else if (c.child_status === 'skipped') { + setStatus(c.sub_issue_id, 'blocked'); + } + } + + // 3. succeeded/released nodes are untouched (never re-run landed work). + + // 4. Release every now-ready node whose predecessors are ALL succeeded. + const toRelease: string[] = []; + for (const c of children) { + if (statusOf.get(c.sub_issue_id) !== 'ready') continue; + const allSucceeded = c.depends_on.every((dep) => statusOf.get(dep) === 'succeeded'); + if (allSucceeded) toRelease.push(c.sub_issue_id); + } + + return { statusUpdates: updates, toRelease, failedCount, skippedCount, succeededCount }; +} diff --git a/cdk/src/handlers/shared/orchestration-release.ts b/cdk/src/handlers/shared/orchestration-release.ts new file mode 100644 index 000000000..bea88bd17 --- /dev/null +++ b/cdk/src/handlers/shared/orchestration-release.ts @@ -0,0 +1,929 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Child-task release for orchestration. + * + * The single path that turns an orchestration child row into a running + * ABCA task. Used in two places: + * - seed time (the webhook processor / discovery): release the root + * children (layer 0) so the graph starts. + * - reconcile time (the TaskTable-stream reconciler): release children + * whose predecessors just all succeeded. + * + * Each release: + * 1. createTaskCore(...) with channelSource 'linear' + orchestration + * metadata, idempotency-keyed on ``orchestration_id#sub_issue_id`` + * so a duplicate stream event / webhook replay never double-creates. + * 2. on 201, conditionally flip the row child_status blocked|ready → + * released and stamp child_task_id (the GSI then resolves the + * task back to its row on the child's terminal event). + * + * The conditional update (``child_status IN (blocked, ready)``) is the + * second idempotency guard: if two reconcile invocations race the same + * release, only one wins the status flip; createTaskCore's own + * idempotency key means the loser doesn't create a second task either. + */ + +import { + type DynamoDBDocumentClient, + GetCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; +import type { createTaskCore as CreateTaskCoreFn } from './create-task-core'; +import { logger } from './logger'; +import { selectBaseBranch } from './orchestration-base-branch'; +import { isIntegrationNode } from './orchestration-integration-node'; +import { computeReconcilePlan } from './orchestration-reconcile'; +import type { + ChildStatus, + OrchestrationChildRow, + OrchestrationReleaseContext, +} from './orchestration-store'; +import type { AttachmentRecord, ChannelSource } from './types'; +import { MAX_ATTACHMENTS_PER_TASK } from './validation'; +import { CODING_WORKFLOW_ID } from './workflows'; + +/** + * The trigger channel an orchestration runs under. Defaults to ``'linear'`` + * everywhere (the only wired trigger today + back-compat for meta rows + * seeded before the field existed). Part of the trigger-agnostic seam. + */ +const DEFAULT_ORCHESTRATION_CHANNEL: ChannelSource = 'linear'; + +/** + * Read a user's free concurrency budget (``cap - active_count``) so a + * release pass throttles to it instead of over-releasing children that admission + * control would then hard-fail. Best-effort: on any read error returns the full + * ``cap`` (degrade to today's release-all behavior rather than stall the + * orchestration — admission control is still the backstop). Never negative. + * + * NOTE this is an INSTANTANEOUS snapshot — between this read and the child task's + * own admission attempt, other tasks may start. That race is fine: admission + * control remains the hard ceiling; throttling here just keeps the common case + * (a wide fan-out releasing into an empty/quiet user) from mass-failing. A child + * that still loses a tighter race is left ``ready`` and retried (it is not over the + * cap-as-guillotine path because the throttle keeps the batch small). + */ +export async function readConcurrencyBudget( + ddb: DynamoDBDocumentClient, + concurrencyTableName: string, + userId: string, + maxConcurrent: number, +): Promise { + try { + const res = await ddb.send(new GetCommand({ + TableName: concurrencyTableName, + Key: { user_id: userId }, + ProjectionExpression: 'active_count', + })); + const active = Number(res.Item?.active_count ?? 0); + return Math.max(0, maxConcurrent - (Number.isFinite(active) ? active : 0)); + } catch (err) { + logger.warn('Concurrency-budget read failed — releasing without throttle (admission still gates)', { + user_id: userId, + error: err instanceof Error ? err.message : String(err), + }); + return maxConcurrent; + } +} + +export interface ReleaseChildParams { + readonly ddb: DynamoDBDocumentClient; + readonly tableName: string; + /** The orchestration child row to release. */ + readonly row: OrchestrationChildRow; + /** Platform user the child task is attributed to (parent's submitter). */ + readonly platformUserId: string; + /** Linear OAuth secret ARN + slug for the agent's outbound Linear GraphQL + * (reactions/state via linear_reactions.py — there is no Linear MCP). */ + readonly linearOauthSecretArn?: string; + readonly linearWorkspaceSlug?: string; + readonly linearProjectId?: string; + /** The base branch this child stacks on. Absent → root (off main). */ + readonly baseBranch?: string; + /** + * Predecessor branches to merge into the child's branch before work (the + * diamond case: two predecessors converging). Absent/empty for root + + * linear children. + */ + readonly mergeBranches?: readonly string[]; + /** Injected createTaskCore (real handler in prod, mock in tests). */ + readonly createTaskCore: typeof CreateTaskCoreFn; + /** + * Parent-issue attachments (screened + stored once at seed time), inherited by + * every child so the coding agent sees the parent's attached spec. + * Passed to createTaskCore's preScreenedAttachments seam. The PARENT owns these + * S3 objects — a child failure must NOT delete them (createTaskCore only rolls + * back its own inline uploads, of which children have none). + */ + readonly preScreenedAttachments?: readonly AttachmentRecord[]; + /** + * The parent epic's title/body, inherited by every child so PARALLEL siblings + * work from one shared statement of names and shapes. + * + * Without it each child sees only its own sub-issue text and invents its own + * contract. That is not hypothetical: an API child and a UI child of the same + * epic shipped `kyoto`/`checkIn` against `wander-kyoto`/`startDate`, each passed + * its own tests, merged without conflict, and produced a broken flow. The + * agreement existed only in the parent, which neither could see. + */ + readonly parentContext?: { + readonly title?: string; + readonly description?: string; + }; + /** ISO timestamp (injected for testability). */ + readonly now: string; + /** + * Trigger channel the child task is created under. Defaults to ``'linear'``. + * Threaded from the orchestration's release context so a non-Linear trigger + * attributes its children to the right plane. Part of the trigger-agnostic seam. + */ + readonly channelSource?: ChannelSource; + /** + * Epic RETRY: when true, the idempotency key is salted with the + * child's PRIOR ``child_task_id`` so a re-run creates a genuinely NEW task + * rather than idempotently replaying the failed one. The prior task id is + * distinct per retry round (each round's prior task differs) yet stable within + * a round (a webhook redelivery of the same retry sees the same prior id → + * one new task, not many). A first release (no prior task id) is unaffected — + * the key stays the back-compat ``orch_sub``. Without this, a retry flips the + * row to ``released`` but ``createTaskCore`` returns the OLD failed task (200 + * idempotent replay) and nothing actually re-runs — observed in practice on a + * retried epic. + */ + readonly retry?: boolean; +} + +export type ReleaseChildResult = + | { readonly kind: 'released'; readonly taskId: string } + | { readonly kind: 'create_failed'; readonly statusCode: number; readonly body: string } + // The create failed DETERMINISTICALLY (a 4xx that + // will fail identically on every retry — e.g. a guardrail/content-policy block + // or a validation error), so the child was marked terminally ``failed`` rather + // than rolled back to ``ready``. Distinct from ``create_failed`` (transient, + // stays ``ready`` for the next sweep) so the caller settles the epic + // finished-with-failures instead of re-attempting a doomed create forever. + | { readonly kind: 'create_failed_terminal'; readonly statusCode: number; readonly body: string; readonly failureReason: string } + | { readonly kind: 'already_released' } + | { readonly kind: 'error'; readonly message: string }; + +/** + * A {@link ReleaseChildResult} tagged with the child it came from, returned by + * {@link releaseReadyChildren} so a caller can correlate a result back to a row + * (e.g. patch a terminally-failed child into the in-memory view before settling). + */ +export type ReleaseChildReadyResult = ReleaseChildResult & { readonly subIssueId: string }; + +// The status codes createTaskCore ACTUALLY returns on a non-success (verified +// against create-task-core.ts, not assumed from HTTP-code lore): 400 +// VALIDATION_ERROR (incl. the guardrail block), 409 DUPLICATE_TASK (idempotent +// replay), 422 REPO_NOT_ONBOARDED, 500/503 server/service errors. There is no +// 403/404/408/429 path here. +const HTTP_CONFLICT = 409; // idempotent replay — a task already exists for this key +const HTTP_CLIENT_ERROR_MIN = 400; +const HTTP_SERVER_ERROR_MIN = 500; + +/** + * Is a create-task failure DETERMINISTIC (won't succeed on a bare retry — it + * needs someone to change something first) vs TRANSIENT (a later sweep could + * succeed on its own)? This decides terminal-fail (settle the epic) vs + * rollback-to-``ready`` (let the scheduled sweep retry). Getting it wrong in the + * rollback direction is an INFINITE silent strand: the stranded-orchestration + * sweep re-releases a ``ready`` child forever and never terminally-fails it. + * + * Against the codes createTaskCore actually emits: + * - 400 (validation / guardrail block) and 422 (repo not onboarded) → + * DETERMINISTIC. Neither self-heals; the user must edit/reword the sub-issue + * or onboard the repo, THEN re-run via ``@bgagent retry``. Rolling back would + * loop the sweep forever. + * - 409 (duplicate/idempotent replay) → NOT a real failure: a task already + * exists for this key, so treat like a transient (roll back; a re-release + * idempotent-replays to 200 and finalizes). Never terminal. + * - 5xx (server/service) → TRANSIENT; a later sweep can succeed on its own. + */ +function isDeterministicCreateFailure(statusCode: number): boolean { + if (statusCode === HTTP_CONFLICT) return false; + return statusCode >= HTTP_CLIENT_ERROR_MIN && statusCode < HTTP_SERVER_ERROR_MIN; +} + +const HTTP_UNPROCESSABLE = 422; // REPO_NOT_ONBOARDED + +/** + * A short, user-facing reason for a deterministic create failure, shown as + * the child's panel ❌ sub-line (the child never got a task, so there's no + * error_message to read). Keys on the STATUS CODE first (reliable + owned by + * this layer) so the message can't silently degrade when an upstream reword + * changes the body prose; the body substring is only a refinement to name the + * guardrail case specifically (a 400 is either a guardrail block or a plain + * validation error, and the code alone doesn't tell them apart). Every reason + * ends with the same next step — fix the cause, then ``@bgagent retry``. + */ +function deterministicFailureReason(statusCode: number, body: string): string { + const retry = 'then reply `@bgagent retry` on the epic to re-run.'; + if (statusCode === HTTP_UNPROCESSABLE) { + return `Couldn't start — this repo isn't onboarded to ABCA. Onboard it, ${retry}`; + } + // 400: distinguish a guardrail/content-policy block (rewordable) from other validation. + if (/content policy|guardrail/i.test(body || '')) { + return `Blocked by content policy — reword this sub-issue, ${retry}`; + } + return `Couldn't start (validation error) — edit this sub-issue, ${retry}`; +} + +/** Build the child task description from the sub-issue's identifier/title. */ +/** + * The shared-context block prepended to every child's task description. + * + * Delimited and explicitly labelled so the agent can tell the epic-wide agreement + * from its OWN task, and told plainly that siblings are working in parallel from + * this same text — which is the part that makes it act on it rather than treat it + * as background. Names and shapes stated here are to be used verbatim rather than + * re-invented, because a sibling has already been told the same thing. + * + * Empty when the epic has no context (an older row, or a bare parent) so a child + * prompt is never padded with an empty heading. + */ +function parentContextBlock( + parentContext?: { readonly title?: string; readonly description?: string }, +): string[] { + const title = parentContext?.title?.trim(); + const description = parentContext?.description?.trim(); + if (!title && !description) return []; + // DESCRIPTIVE, not imperative — deliberately, and verified against the live + // guardrail. An instruction-shaped preamble ("use it exactly", "do not invent") + // stacks with the epic's own imperatives and the whole task description trips the + // PROMPT_ATTACK filter at MEDIUM confidence, which fails the child at admission + // with "blocked by content policy". Reproduced: the imperative wording blocks, this + // wording passes, with the same epic text after it. So the constraint is conveyed + // by stating what the siblings are already doing and what disagreeing with them + // would mean — which the model acts on just as well and the screen accepts. + return [ + '--- SHARED ORCHESTRATION CONTEXT (the parent epic) ---', + 'Several sub-issues of this epic are being built in parallel, and each was given', + 'the text below. The names, routes, fields and shapes it states are shared with', + 'those siblings; an alternative chosen here would not match theirs.', + '', + ...(title ? [`Epic: ${title}`, ''] : []), + ...(description ? [description, ''] : []), + '--- END SHARED ORCHESTRATION CONTEXT ---', + '', + ]; +} + +function buildChildDescription( + row: OrchestrationChildRow, + parentContext?: { readonly title?: string; readonly description?: string }, +): string { + // The synthetic integration node has no real sub-issue / feature + // work — its job is to merge all leaf branches (already merged into its + // branch by repo.py's predecessor-merge) into one combined result. Give + // the agent a merge-focused instruction rather than a feature prompt. + if (isIntegrationNode(row.sub_issue_id)) { + return [ + ...parentContextBlock(parentContext), + 'Integrate the completed sub-issue branches into one combined result.', + '', + "All predecessor sub-issue branches have already been merged into this task's", + 'branch before you started. Your job:', + '- Resolve any merge conflicts left in the working tree.', + '- Ensure the combined result builds and existing tests pass (run the build/tests).', + '- Do NOT add new features — this is an integration/merge task only.', + '- Open a PR with the combined result so the epic has a single reviewable artifact.', + ].join('\n'); + } + const block = parentContextBlock(parentContext); + // ONE part: the caller joins parts with a blank line, which would double-space + // every line inside the block and break the delimiters' visual grouping. + const parts: string[] = block.length > 0 ? [block.join('\n').trimEnd()] : []; + if (row.display_id && row.title) { + parts.push(`${row.display_id}: ${row.title}`); + } else if (row.title) { + parts.push(row.title); + } else if (row.display_id) { + parts.push(row.display_id); + } + // Include the planner's scope below the title when it adds detail. The + // reviewer approved a plan that may name a concrete deliverable (a filename, a + // route); the coding agent must SEE it or it builds a title-only guess and the + // plan's promise breaks (observed in practice: the plan said dashboard.html, + // the agent shipped team-dashboard.html → 404). Skip when the description just + // echoes the title. + const desc = (row.description ?? '').trim(); + if (desc && desc !== row.title) parts.push(desc); + return parts.join('\n\n') || `Linear sub-issue ${row.sub_issue_id}`; +} + +/** + * Release one orchestration child as an ABCA task. Idempotent: a + * duplicate call (stream redelivery, racing reconcile) does not create a + * second task, and the row flip to ``released`` is conditional. + */ +export async function releaseChild(params: ReleaseChildParams): Promise { + const { ddb, tableName, row, platformUserId, baseBranch, createTaskCore, now } = params; + const channelSource = params.channelSource ?? DEFAULT_ORCHESTRATION_CHANNEL; + + // Surface-NEUTRAL keys: every released child carries these whatever seeded it. + // The reconciler maps a terminal task back to its row via the orchestration + // pair, so it must not depend on any one surface's metadata. + const channelMetadata: Record = { + orchestration_id: row.orchestration_id, + orchestration_sub_issue_id: row.sub_issue_id, + }; + + // Surface-SPECIFIC keys, only for the surface that actually seeded this + // orchestration. Stamping them unconditionally would hand a non-Linear child a + // set of Linear ids naming rows in a workspace it has nothing to do with — + // harmless only because every consumer happens to check ``channel_source`` + // first, which is a coincidence to rely on rather than a contract. + if (channelSource === 'linear') { + channelMetadata.linear_workspace_id = row.credentials_ref; + // Provenance only — nothing reads this back today, but it's been on every + // released child since the executor shipped, so it stays rather than being + // quietly dropped as part of a naming change. + channelMetadata.parent_linear_issue_id = row.parent_issue_ref; + // Only set linear_issue_id (the agent's reaction/comment target) for a REAL + // sub-issue. A synthetic integration node has no issue behind it — passing + // its id would make the agent's reaction call 4xx. Omitting it lets the + // agent skip reactions cleanly. + if (!isIntegrationNode(row.sub_issue_id)) { + channelMetadata.linear_issue_id = row.sub_issue_id; + } + if (row.display_id) channelMetadata.linear_issue_identifier = row.display_id; + if (params.linearProjectId) channelMetadata.linear_project_id = params.linearProjectId; + if (params.linearOauthSecretArn) channelMetadata.linear_oauth_secret_arn = params.linearOauthSecretArn; + if (params.linearWorkspaceSlug) channelMetadata.linear_workspace_slug = params.linearWorkspaceSlug; + } + // Stacked base branch + (diamond) predecessor merge-list. The + // orchestrator reads these to set the agent payload's base_branch + + // merge_branches. Absent for roots (agent branches off main as today). + if (params.baseBranch) channelMetadata.orchestration_base_branch = params.baseBranch; + if (params.mergeBranches && params.mergeBranches.length > 0) { + channelMetadata.orchestration_merge_branches = JSON.stringify(params.mergeBranches); + } + + // Deterministic idempotency key: same child never creates two tasks. + // Separator is '_' (NOT '#') because createTaskCore validates the key + // against /^[a-zA-Z0-9_-]{1,128}$/ — a '#' is rejected with a 400 and + // the child silently never starts. orchestration_id (orch_<32hex>) + + // '_' + sub_issue_id (a UUID, all hyphens) stays within 128 chars and + // inside the allowed charset. + // + // RETRY REPLAY FIX: salt the key with the prior + // child_task_id WHENEVER the row already carries one — NOT only when the + // caller passes retry=true. A child being (re-)released while it still has a + // child_task_id inherently means that prior task is TERMINAL (a live child is + // in 'released', never back in blocked/ready), so replaying it under the same + // key would return the OLD failed task (200) and nothing runs. This bit a + // dependency chain A→B: the epic-retry salts A (layer 0, retry=true) but B is + // reset to blocked and later released by the reconciler with retry=false → the + // unsalted key replayed B's dead task. Salting on the id's PRESENCE fixes both + // the immediate layer and the downstream cascade. The prior id (a ULID, 26 + // alnum chars) keeps the key inside the charset; orch_<32> + _ + + + // _ + ≈ 100 chars, under the 128 cap. A first release (no prior id) + // is unchanged. Redelivery-safe: same prior id → same key → one new task. + const baseKey = `${row.orchestration_id}_${row.sub_issue_id}`; + const idempotencyKey = row.child_task_id + ? `${baseKey}_${row.child_task_id}` + : baseKey; + + // EXACTLY-ONCE FIX: flip-then-create. The prior design + // was create-then-flip — createTaskCore ran FIRST (irreversible: mints a task + // + fire-and-forget invokes the orchestrator), and only afterward a conditional + // row flip "deduped". But createTaskCore's own idempotency is an + // eventually-consistent GSI read-then-write, so two concurrent releasers (the + // live TaskTable-stream reconciler AND the scheduled stranded sweep) could BOTH + // pass the in-memory status check, BOTH createTaskCore before either flip + // committed, and BOTH miss the other's not-yet-propagated write → two ECS + // agents + two PRs for one sub-issue. (Analysed in + // ``docs/research/orchestration-reconciler-correctness.md``.) + // + // Now we ATOMICALLY CLAIM the row (blocked|ready → releasing) BEFORE creating + // the task. Only the invocation that wins this conditional Update proceeds to + // createTaskCore; a racing releaser gets ConditionalCheckFailed and returns + // already_released WITHOUT creating a task — the claim is the single + // serialization point, now correctly ahead of the irreversible create. Do this + // FIRST so a losing releaser bails before doing any attachment work below. + try { + await ddb.send(new UpdateCommand({ + TableName: tableName, + Key: { orchestration_id: row.orchestration_id, sub_issue_id: row.sub_issue_id }, + UpdateExpression: 'SET child_status = :releasing, updated_at = :now', + ConditionExpression: 'child_status IN (:blocked, :ready)', + ExpressionAttributeValues: { + ':releasing': 'releasing', + ':now': now, + ':blocked': 'blocked', + ':ready': 'ready', + }, + })); + } catch (err) { + if (isConditionalCheckFailed(err)) { + logger.info('Orchestration child already claimed by a racing releaser (flip-then-create)', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + }); + return { kind: 'already_released' }; + } + logger.error('Failed to claim orchestration child for release', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + return { kind: 'error', message: err instanceof Error ? err.message : String(err) }; + } + + // Attachments a feature child receives: the PARENT epic's spec (inherited via + // release_context) PLUS this sub-issue's OWN attachments (a mockup + // attached to just this piece, hydrated at seed onto the child row). Merge both, + // de-duped by attachment_id (a file both on the epic and the sub-issue isn't + // passed twice). Integration nodes are a pure branch merge — they need neither. + // + // OWN attachments are listed FIRST so that, when the merged set would exceed the + // per-task cap, the child's own files (most relevant to THIS piece) survive and + // the shared epic spec is what gets trimmed. Capped at MAX_ATTACHMENTS_PER_TASK + // with a loud log (never a silent truncation) — each source is independently + // ≤10, so the merge can only overflow a pathological many-file epic+child. + const inheritedAttachments = params.preScreenedAttachments ?? []; + const ownAttachments = row.pre_screened_attachments ?? []; + const dedupedAttachments: AttachmentRecord[] = []; + const seenAttachmentIds = new Set(); + for (const rec of [...ownAttachments, ...inheritedAttachments]) { + if (seenAttachmentIds.has(rec.attachment_id)) continue; + seenAttachmentIds.add(rec.attachment_id); + dedupedAttachments.push(rec); + } + const mergedAttachments = dedupedAttachments.slice(0, MAX_ATTACHMENTS_PER_TASK); + if (dedupedAttachments.length > MAX_ATTACHMENTS_PER_TASK) { + // BACKSTOP only: the webhook processor caps a child's OWN attachments against + // the inherited count at stamp time AND posts a user-visible note there. + // Reaching here means the merged set still overflowed (e.g. the + // epic's own attachment set is large) — keep own-first ordering and log; this + // is not the primary user-facing path, so a warn is the right level. + logger.warn('Child attachment set over the per-task cap at release — trimming (parent-spec files dropped first; user already notified at stamp time)', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + own_count: ownAttachments.length, + inherited_count: inheritedAttachments.length, + merged_count: dedupedAttachments.length, + kept: mergedAttachments.length, + cap: MAX_ATTACHMENTS_PER_TASK, + }); + } + + let result; + try { + result = await createTaskCore( + { + repo: row.repo, + task_description: buildChildDescription(row, params.parentContext), + // Pin the disciplined coding workflow, as every other channel does at its + // own call site. Without it a repo-bound child falls back to the + // repo-OPTIONAL default/agent-v1, whose setup skips the stacked-base and + // predecessor-merge path entirely. + // + // A chain survives that by luck — its child needs no merge, so the agent + // works from the base it was given. A DIAMOND does not: the integration + // node was handed two predecessor branches to merge, got a clean default + // branch instead, and the agent hand-wrote an approximation of one arm. + // The result looked plausible and silently dropped the other arm's work. + workflow_ref: CODING_WORKFLOW_ID, + }, + { + userId: platformUserId, + channelSource, + channelMetadata, + idempotencyKey, + // Parent spec + this child's own attachments. Integration nodes are a pure + // merge of already-built branches, so they don't need the spec — only real + // feature children do. Records reference existing S3 objects (read-only). + ...(mergedAttachments.length > 0 + && !isIntegrationNode(row.sub_issue_id) + && { preScreenedAttachments: mergedAttachments }), + }, + // requestId — reuse the idempotency key for trace correlation. + idempotencyKey, + ); + } catch (err) { + logger.error('Orchestration child createTaskCore threw', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + // We won the claim (row is now 'releasing') but createTaskCore threw — roll + // the claim BACK to 'ready' so a later reconcile/sweep can retry the release, + // instead of leaving the child stuck in the transient 'releasing' state + // forever. Conditional on still being 'releasing' (don't clobber a concurrent + // transition). Best-effort — a failed rollback still gets swept by the + // stranded-orchestration reconciler. + await rollbackClaim(ddb, tableName, row, now); + return { kind: 'error', message: err instanceof Error ? err.message : String(err) }; + } + + // 201 = created; 200 = idempotent replay (task already existed). Both + // mean "a task exists for this child" — treat alike. + if (result.statusCode !== 201 && result.statusCode !== 200) { + // Log the RESPONSE BODY, not just the status — a bare "status:400" + // forces log-archaeology to find the cause (e.g. a rejected + // idempotency key, an un-onboarded repo, a guardrail block). The + // body carries the user-readable error message and code. + logger.warn('Orchestration child task creation returned non-success', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + repo: row.repo, + status: result.statusCode, + response_body: result.body, + idempotency_key: idempotencyKey, + }); + // Split deterministic from transient failures. + // A deterministic 4xx (guardrail/content-policy block, validation error) + // fails identically on every retry — rolling back to 'ready' strands the + // child in an infinite silent re-attempt loop (10-min sweep forever, no + // terminal state, no ❌, epic stuck 👀). Mark it terminally 'failed' so the + // reconcile settles the epic finished-with-failures and the child gets a ❌ + // + a reason (posted by the caller's terminal path). Only TRANSIENT failures + // (5xx / throttle) roll back to 'ready' for a later retry. + if (isDeterministicCreateFailure(result.statusCode)) { + const failureReason = deterministicFailureReason(result.statusCode, result.body); + await failClaimTerminal(ddb, tableName, row, now, failureReason); + return { kind: 'create_failed_terminal', statusCode: result.statusCode, body: result.body, failureReason }; + } + // Claim won but the create failed transiently (5xx, throttle) — roll the + // claim back to 'ready' so the next reconcile/sweep retries it, rather than + // stranding the child in 'releasing'. Note 422 (repo not onboarded) is NOT + // here: it is deterministic and handled above, since a repo doesn't onboard + // itself and retrying would strand the child silently forever. + await rollbackClaim(ddb, tableName, row, now); + return { kind: 'create_failed', statusCode: result.statusCode, body: result.body }; + } + + const { taskId, branchName } = extractTaskIdAndBranch(result.body); + + // Finalize the claim: flip 'releasing' → 'released' and stamp the task id + + // branch. We already hold the claim (won the conditional above), so this is + // conditional only on still being 'releasing' — defensive against a concurrent + // cancel/skip that legitimately moved the row on (in which case we do NOT + // overwrite it; the created task's own terminal event reconciles). + // + // Persist the child's branch_name so a DEPENDENT child's release can + // stack on / merge it (selectBaseBranch reads predecessor branch names). + try { + await ddb.send(new UpdateCommand({ + TableName: tableName, + Key: { orchestration_id: row.orchestration_id, sub_issue_id: row.sub_issue_id }, + UpdateExpression: + 'SET child_status = :released, child_task_id = :tid, child_branch_name = :bn, updated_at = :now', + ConditionExpression: 'child_status = :releasing', + ExpressionAttributeValues: { + ':released': 'released', + ':tid': taskId, + ':bn': branchName, + ':now': now, + ':releasing': 'releasing', + }, + })); + } catch (err) { + if (isConditionalCheckFailed(err)) { + // The row moved off 'releasing' between our claim and finalize (a cancel + // or skip landed). The task was created (idempotency-keyed), so its own + // terminal event will reconcile — we just don't clobber the new state. + logger.info('Orchestration child left releasing state before finalize (concurrent transition)', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + task_id: taskId, + }); + return { kind: 'released', taskId }; + } + logger.error('Failed to mark orchestration child released', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + return { kind: 'error', message: err instanceof Error ? err.message : String(err) }; + } + + logger.info('Orchestration child released', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + task_id: taskId, + base_branch: baseBranch ?? 'main', + }); + return { kind: 'released', taskId }; +} + +/** + * Release a batch of child rows (the ``ready`` ones), using a shared + * release context (from the meta row). Used both at seed time (release + * roots) and by the reconciler (release newly-unblocked dependents). + * + * Each child is released independently; one failure does not abort the + * rest (a transient create failure for child A shouldn't strand B). The + * caller logs/handles per-child results — a ``create_failed`` row stays + * ``ready`` and is retried on the next reconcile pass. + */ +export async function releaseReadyChildren( + ddb: DynamoDBDocumentClient, + tableName: string, + rows: readonly OrchestrationChildRow[], + releaseContext: OrchestrationReleaseContext, + createTaskCore: typeof CreateTaskCoreFn, + now: string, + /** + * The FULL child set (not just the releasable subset), so a + * child's base branch can be derived from its predecessors' persisted + * ``child_branch_name``. Defaults to ``rows`` for back-compat with + * callers that pass the full set as ``rows`` and release roots (roots + * have no predecessors, so selection degrades to off-main). + */ + allChildren?: readonly OrchestrationChildRow[], + /** Repo default branch for roots + diamond bases. Defaults to 'main'. */ + defaultBranch = 'main', + /** + * Max children to actually release this pass — the user's free + * concurrency budget (``cap - active_count``). When set, only this many + * ``ready`` children are released; the rest are LEFT ``ready`` (a no-op, + * not a failure) for a later reconcile pass to pick up as slots free. + * ``undefined`` = release all (back-compat; callers that don't throttle). + * A value ``<= 0`` releases nothing this pass. + */ + maxToRelease?: number, + /** + * Epic RETRY: salt each child's idempotency key with its prior + * ``child_task_id`` so a re-run spawns a NEW task instead of replaying the + * failed one. Only the epic-retry path passes true; every other caller + * (seed, extend, forward cascade, recovery) omits it → back-compat key. + */ + retry = false, +): Promise { + const all = allChildren ?? rows; + const branchOf = new Map( + all.filter((c) => c.child_branch_name).map((c) => [c.sub_issue_id, c.child_branch_name as string]), + ); + // Throttle to the available budget. Sort by sub_issue_id for a + // deterministic, fair release order across passes. Releasing fewer than + // are ready is intentional — the leftovers stay ``ready`` and the next + // reconcile (sibling completion) or the scheduled sweep releases them. + const ready = rows.filter((r) => r.child_status === 'ready'); + const releasable = maxToRelease === undefined + ? ready + : [...ready].sort((a, b) => a.sub_issue_id.localeCompare(b.sub_issue_id)).slice(0, Math.max(0, maxToRelease)); + if (maxToRelease !== undefined && releasable.length < ready.length) { + logger.info('Orchestration release throttled to concurrency budget', { + ready: ready.length, + releasing: releasable.length, + budget: maxToRelease, + }); + } + const results: ReleaseChildReadyResult[] = []; + for (const row of releasable) { + // Derive the base from this child's predecessors' persisted branches. + const selection = selectBaseBranch({ + predecessors: row.depends_on.map((sub) => ({ + sub_issue_id: sub, + branch_name: branchOf.get(sub) ?? '', + })), + defaultBranch, + }); + const childResult = await releaseChild({ + ddb, + tableName, + row, + platformUserId: releaseContext.platform_user_id, + ...(releaseContext.linear_oauth_secret_arn !== undefined && { + linearOauthSecretArn: releaseContext.linear_oauth_secret_arn, + }), + ...(releaseContext.linear_workspace_slug !== undefined && { + linearWorkspaceSlug: releaseContext.linear_workspace_slug, + }), + ...(releaseContext.linear_project_id !== undefined && { + linearProjectId: releaseContext.linear_project_id, + }), + // Trigger-agnostic: carry the orchestration's channel onto the + // child. ``releaseChild`` defaults to 'linear' when absent. + ...(releaseContext.channel_source !== undefined && { + channelSource: releaseContext.channel_source as ChannelSource, + }), + // Every child inherits the parent's screened attachments. + ...(releaseContext.pre_screened_attachments !== undefined && { + preScreenedAttachments: releaseContext.pre_screened_attachments, + }), + // …and the epic's own words, so siblings released in the SAME batch (the + // parallel case) are working from one shared contract rather than each + // inferring its own from its sub-issue title. + ...(releaseContext.parent_context !== undefined && { + parentContext: releaseContext.parent_context, + }), + // Root → 'main' base, no merges (omit so today's off-main behavior + // is unchanged). Linear → predecessor branch. Diamond → main + merges. + ...(selection.shape !== 'root' && { baseBranch: selection.base_branch }), + ...(selection.merge_branches.length > 0 && { mergeBranches: selection.merge_branches }), + createTaskCore, + now, + retry, + }); + // Correlate the result to its child so callers can patch their in-memory + // view: a terminally-failed child must be reflected in the settle that + // runs right after this release, since no later stream event will re-drive + // the reconciler for a child that never became a task. + results.push({ ...childResult, subIssueId: row.sub_issue_id }); + } + return results; +} + +/** Pull task_id + branch_name out of a createTaskCore success body (best-effort). */ +function extractTaskIdAndBranch(body: string): { taskId: string; branchName: string } { + try { + const parsed = JSON.parse(body) as { + data?: { task_id?: string; branch_name?: string }; + task_id?: string; + branch_name?: string; + }; + return { + taskId: parsed.data?.task_id ?? parsed.task_id ?? '', + branchName: parsed.data?.branch_name ?? parsed.branch_name ?? '', + }; + } catch { + return { taskId: '', branchName: '' }; + } +} + +/** + * Roll a claimed-but-not-created child back from the transient 'releasing' state + * to 'ready' so a later reconcile/sweep can retry the release. Conditional on the + * row still being 'releasing' (never clobber a concurrent transition). Best-effort: + * if the rollback itself fails, the stranded-orchestration sweep still recovers a + * child left in 'releasing' (it is treated as an in-flight release with no task id). + * Preserves child_task_id so the retry-replay salt still applies on the next try. + */ +async function rollbackClaim( + ddb: DynamoDBDocumentClient, + tableName: string, + row: OrchestrationChildRow, + now: string, +): Promise { + try { + await ddb.send(new UpdateCommand({ + TableName: tableName, + Key: { orchestration_id: row.orchestration_id, sub_issue_id: row.sub_issue_id }, + UpdateExpression: 'SET child_status = :ready, updated_at = :now', + ConditionExpression: 'child_status = :releasing', + ExpressionAttributeValues: { ':ready': 'ready', ':releasing': 'releasing', ':now': now }, + })); + } catch (err) { + if (isConditionalCheckFailed(err)) return; // already moved on — fine + logger.warn('Failed to roll back orchestration child claim (sweep will recover)', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + } +} + +/** + * Flip a claimed ('releasing') child to terminal 'failed' after a + * DETERMINISTIC create failure (guardrail/validation) so it does NOT re-attempt + * forever. Mirrors {@link rollbackClaim} but targets 'failed' instead of 'ready' + * — the reconcile/sweep then settles the epic finished-with-failures and posts + * the child's ❌. Conditional on still being 'releasing' (don't clobber a + * concurrent cancel/skip). Best-effort: a failed write is swept by the + * stranded-orchestration reconciler (which will see a 'releasing' row aged out). + */ +async function failClaimTerminal( + ddb: DynamoDBDocumentClient, + tableName: string, + row: OrchestrationChildRow, + now: string, + failureReason: string, +): Promise { + try { + await ddb.send(new UpdateCommand({ + TableName: tableName, + Key: { orchestration_id: row.orchestration_id, sub_issue_id: row.sub_issue_id }, + UpdateExpression: 'SET child_status = :failed, failure_reason = :reason, updated_at = :now', + ConditionExpression: 'child_status = :releasing', + ExpressionAttributeValues: { + ':failed': 'failed', + ':reason': failureReason, + ':releasing': 'releasing', + ':now': now, + }, + })); + logger.warn('Orchestration child marked terminally failed (deterministic create failure)', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + }); + } catch (err) { + if (isConditionalCheckFailed(err)) return; // already moved on — fine + logger.warn('Failed to mark orchestration child terminally failed (sweep will recover)', { + orchestration_id: row.orchestration_id, + sub_issue_id: row.sub_issue_id, + error: err instanceof Error ? err.message : String(err), + }); + } +} + +function isConditionalCheckFailed(err: unknown): boolean { + return ( + typeof err === 'object' + && err !== null + && 'name' in err + && (err as { name?: string }).name === 'ConditionalCheckFailedException' + ); +} + +/** + * Persist the transitive skips caused by a child that failed DETERMINISTICALLY at + * release time, and return the child view patched with the failure + skips. + * + * A guardrail/validation rejection means the child never became a task, so no task + * event will ever wake the reconciler for it. ``releaseReadyChildren`` already + * wrote ``child_status='failed'`` + ``failure_reason`` for the node itself, but a + * mid-graph failure must ALSO transitively skip its dependents — exactly as a + * task-level failure does — or those dependents stay ``blocked`` forever and the + * epic never reaches all-terminal. + * + * Both release call sites need this, which is why it lives here rather than in one + * handler: the reconciler releases children as predecessors succeed, and the + * SEED path releases the roots. A guardrail rejection on a root at seed time used + * to leave the epic at "🔄 N/M" with no settle until the 10-minute stranded sweep + * swept it up, because only the reconciler had this step (observed in practice). + * + * Reuses {@link computeReconcilePlan} — the same BFS over reverse dependencies the + * task-failure path uses — so a guardrail failure and a task failure skip + * dependents identically. + * + * Idempotent: each skip write is guarded on ``child_status IN (blocked, ready)``, + * so it never stamps a released/running sibling terminal. + * + * Returns ``children`` untouched when nothing failed terminally, so callers can + * apply it unconditionally. + */ +export async function applyTerminalCreateFailures( + ddb: DynamoDBDocumentClient, + tableName: string, + orchestrationId: string, + children: readonly OrchestrationChildRow[], + results: readonly ReleaseChildReadyResult[], + now: string, +): Promise { + const failedReasons = new Map(); + for (const r of results) { + if (r.kind === 'create_failed_terminal') failedReasons.set(r.subIssueId, r.failureReason); + } + if (failedReasons.size === 0) return children; + + // Accumulate into one status map so a diamond whose two failed roots feed a + // shared leaf skips that leaf once. + const statusById = new Map(children.map((c) => [c.sub_issue_id, c.child_status])); + for (const failedId of failedReasons.keys()) { + const plan = computeReconcilePlan( + { sub_issue_id: failedId, status: 'FAILED' }, + children.map((c) => ({ ...c, child_status: statusById.get(c.sub_issue_id) ?? c.child_status })), + ); + for (const u of plan.statusUpdates) statusById.set(u.sub_issue_id, u.child_status); + } + + // Persist ONLY the skips — the failed nodes themselves were already written + // (with their failure_reason) by the release path. + for (const [subIssueId, status] of statusById) { + if (status !== 'skipped' || children.find((c) => c.sub_issue_id === subIssueId)?.child_status === 'skipped') { + continue; + } + try { + await ddb.send(new UpdateCommand({ + TableName: tableName, + Key: { orchestration_id: orchestrationId, sub_issue_id: subIssueId }, + UpdateExpression: 'SET child_status = :s, updated_at = :now', + ConditionExpression: 'child_status IN (:blocked, :ready)', + ExpressionAttributeValues: { ':s': 'skipped', ':blocked': 'blocked', ':ready': 'ready', ':now': now }, + })); + } catch (err) { + if (isConditionalCheckFailed(err)) continue; // already moved on — fine + throw err; + } + } + + return children.map((c) => { + if (failedReasons.has(c.sub_issue_id)) { + return { ...c, child_status: 'failed' as const, failure_reason: failedReasons.get(c.sub_issue_id)! }; + } + const s = statusById.get(c.sub_issue_id); + return s === 'skipped' && c.child_status !== 'skipped' ? { ...c, child_status: 'skipped' as const } : c; + }); +} diff --git a/cdk/src/handlers/shared/orchestration-restack.ts b/cdk/src/handlers/shared/orchestration-restack.ts new file mode 100644 index 000000000..dfe836168 --- /dev/null +++ b/cdk/src/handlers/shared/orchestration-restack.ts @@ -0,0 +1,194 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Re-stack planning — pure logic. + * + * When a predecessor sub-issue's PR branch changes after a dependent already + * merged it in, the dependent is STALE. This module computes WHICH dependents + * must be re-stacked and IN WHAT ORDER, given an orchestration snapshot and + * the sub-issue whose branch changed. No I/O — the handler does the GitHub / + * DynamoDB / task-creation side effects using this plan. + * + * Rules: + * - Re-stack only the changed node's TRANSITIVE dependents (everything + * downstream that built on it, directly or indirectly). + * - A dependent is re-stackable only if it has actually started — it carries + * a ``child_task_id`` + ``child_branch_name`` (released). A still-``blocked`` + * dependent will pick up the new predecessor code when it is first + * released, so it needs no re-stack. + * - Order dependents in topological order (a dependent is re-stacked only + * after the predecessors it merges have been re-stacked), so each re-stack + * merges already-current predecessor branches. + * - The changed node itself is NOT re-stacked (its own branch is what changed). + */ + +import type { OrchestrationChildRow } from './orchestration-store'; + +/** A single dependent to re-stack, with the predecessor branches to merge in. */ +export interface RestackStep { + /** The dependent sub-issue row to re-stack. */ + readonly child: OrchestrationChildRow; + /** + * The branches to merge into the dependent's branch — its predecessors' + * CURRENT head branches (the changed node's branch + any sibling + * predecessor branches the dependent also depends on). The agent merges + * these into the existing dependent branch. + */ + readonly mergeBranches: readonly string[]; +} + +const RELEASED_OR_TERMINAL: ReadonlySet = new Set([ + 'released', 'succeeded', 'failed', 'skipped', +]); + +/** + * Compute the ordered re-stack plan for ``changedSubIssueId``. + * + * @param children the orchestration's child rows (full snapshot, excl. meta) + * @param changedSubIssueId the sub-issue whose PR branch changed + * @returns dependents to re-stack, in topological order; empty if none + * (the changed node has no started dependents, or isn't in the graph). + */ +export function planRestack( + children: readonly OrchestrationChildRow[], + changedSubIssueId: string, +): readonly RestackStep[] { + const byId = new Map(children.map((c) => [c.sub_issue_id, c])); + if (!byId.has(changedSubIssueId)) return []; + + // ── 1. Transitive dependents of the changed node (BFS down the DAG). ── + // successorsOf[x] = nodes that depend ON x. + const successors = new Map(); + for (const c of children) { + for (const dep of c.depends_on) { + (successors.get(dep) ?? successors.set(dep, []).get(dep)!).push(c.sub_issue_id); + } + } + const affected = new Set(); + const queue = [...(successors.get(changedSubIssueId) ?? [])]; + while (queue.length > 0) { + const id = queue.shift()!; + if (affected.has(id)) continue; + affected.add(id); + for (const next of successors.get(id) ?? []) queue.push(next); + } + + // ── 2. Keep only dependents that have STARTED (have a branch to re-stack). + // A blocked dependent will see the new code when it is first released. + const restackable = [...affected] + .map((id) => byId.get(id)!) + .filter((c) => c.child_branch_name && RELEASED_OR_TERMINAL.has(c.child_status)); + + // ── 3. Topological order over the affected sub-graph, so a dependent is + // re-stacked after the predecessors it will merge. Kahn over edges among + // the restackable set (+ the changed node as the always-ready source). + const inScope = new Set(restackable.map((c) => c.sub_issue_id)); + const ordered = topoOrder(restackable, inScope); + + // ── 4. For each, the branches to merge = its predecessors' current head + // branches that are in scope (the changed node + affected predecessors). + // The changed node's own branch is included so direct dependents re-merge it. + return ordered.map((child) => { + const mergeBranches = child.depends_on + .filter((dep) => dep === changedSubIssueId || inScope.has(dep)) + .map((dep) => byId.get(dep)?.child_branch_name) + .filter((b): b is string => Boolean(b)); + return { child, mergeBranches }; + }).filter((step) => step.mergeBranches.length > 0); +} + +/** + * Plan the re-stack of a changed node's DIRECT (one-hop) started dependents. + * + * Used by the reconciler-driven cascade: when an + * iteration/restack task on node X completes, we re-stack only the children + * that depend DIRECTLY on X — each of those, when ITS restack task completes, + * re-fires the reconciler and cascades to ITS dependents. Doing one hop per + * completion (rather than ``planRestack``'s whole transitive set at once) is + * what keeps a chain correct: C must re-stack only AFTER B's branch carries + * the new code, not racing B's restack task. + * + * A direct dependent is re-stackable only if it has STARTED (released/terminal + * with a branch). Its merge-list is its predecessors' current head branches — + * the changed node + any sibling predecessors that have a branch — so a + * diamond fan-in re-merges every arm it depends on. + * + * @param children full orchestration child snapshot (excl. meta) + * @param changedSubIssueId the node whose branch just changed + * @returns the direct dependents to re-stack now (deterministic by id); empty + * if none have started. + */ +export function planDirectRestack( + children: readonly OrchestrationChildRow[], + changedSubIssueId: string, +): readonly RestackStep[] { + const byId = new Map(children.map((c) => [c.sub_issue_id, c])); + if (!byId.has(changedSubIssueId)) return []; + + const directDependents = children + .filter((c) => c.depends_on.includes(changedSubIssueId)) + .filter((c) => c.child_branch_name && RELEASED_OR_TERMINAL.has(c.child_status)) + .sort((a, b) => a.sub_issue_id.localeCompare(b.sub_issue_id)); + + return directDependents + .map((child) => { + // Merge every predecessor that currently has a branch — the changed + // node plus any sibling predecessors (so a diamond fan-in re-merges all + // arms, not just the one that changed). + const mergeBranches = child.depends_on + .map((dep) => byId.get(dep)?.child_branch_name) + .filter((b): b is string => Boolean(b)); + return { child, mergeBranches }; + }) + .filter((step) => step.mergeBranches.length > 0); +} + +/** Kahn's algorithm over the in-scope sub-graph (deterministic by id). */ +function topoOrder( + nodes: readonly OrchestrationChildRow[], + inScope: ReadonlySet, +): readonly OrchestrationChildRow[] { + const byId = new Map(nodes.map((c) => [c.sub_issue_id, c])); + const indeg = new Map(); + for (const c of nodes) { + indeg.set(c.sub_issue_id, c.depends_on.filter((d) => inScope.has(d)).length); + } + const ready = nodes + .filter((c) => (indeg.get(c.sub_issue_id) ?? 0) === 0) + .map((c) => c.sub_issue_id) + .sort(); + const out: OrchestrationChildRow[] = []; + while (ready.length > 0) { + const id = ready.shift()!; + out.push(byId.get(id)!); + // decrement successors within scope + for (const c of nodes) { + if (c.depends_on.includes(id)) { + const d = (indeg.get(c.sub_issue_id) ?? 0) - 1; + indeg.set(c.sub_issue_id, d); + if (d === 0) { + ready.push(c.sub_issue_id); + ready.sort(); + } + } + } + } + return out; +} diff --git a/cdk/src/handlers/shared/orchestration-rollup.ts b/cdk/src/handlers/shared/orchestration-rollup.ts new file mode 100644 index 000000000..261bc5563 --- /dev/null +++ b/cdk/src/handlers/shared/orchestration-rollup.ts @@ -0,0 +1,623 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Parent rollup comments for Linear orchestration. + * + * The fan-out plane posts a final-status comment on each CHILD's + * sub-issue. The PARENT issue has no task, so its aggregate rollup is + * posted here, by the reconciler, which already holds the orchestration + * snapshot. The comment renderer is pure (unit-testable); ``postRollup`` + * wraps ``postIssueComment`` best-effort (a failed Linear comment must + * never fail the reconcile — gating is the source of truth). + */ + +import { logger } from './logger'; +import type { Channel, IssueRef } from './orchestration-channel'; +import { isIntegrationNode } from './orchestration-integration-node'; +import { ORCH_LOG } from './orchestration-log-events'; +import type { OrchestrationChildRow } from './orchestration-store'; +import { encodeMarkdownUrl } from './screenshot-url'; +import { DEFAULT_LABEL_FILTER } from './trigger-label'; + +/** Which rollup we're posting — drives the heading + emoji. */ +export type RollupKind = 'complete' | 'partial_failure' | 'cancelled'; + +export interface RollupChildView { + readonly sub_issue_id: string; + readonly display_id?: string; + readonly title?: string; + readonly child_status: string; + readonly child_task_id?: string; + /** + * The child task's PR url, when one was opened. Resolved by the + * reconciler from the TaskTable at rollup time (pr_url lands on the + * TaskRecord in a separate write from the status transition, so it is + * not persisted on the orchestration row). Rendered as a link on the + * child's line; the integration node's PR is additionally surfaced as a + * prominent callout (it is the fan-out's combined deliverable). + */ + readonly pr_url?: string; +} + +const STATUS_ICON: Record = { + succeeded: '✅', + failed: '❌', + skipped: '⏭️', + released: '🔄', + releasing: '🔄', // transient flip-then-create claim — mid-launch + ready: '🔄', + blocked: '⏳', +}; + +/** + * Render the parent rollup comment body (pure). Lists each child with its + * status, and a one-line summary. ``kind`` is derived by the caller from + * the terminal child statuses. + */ +export function renderRollupComment( + kind: RollupKind, + children: readonly RollupChildView[], +): string { + const counts = { succeeded: 0, failed: 0, skipped: 0 }; + for (const c of children) { + if (c.child_status === 'succeeded') counts.succeeded += 1; + else if (c.child_status === 'failed') counts.failed += 1; + else if (c.child_status === 'skipped') counts.skipped += 1; + } + + const heading = + kind === 'complete' + ? '✅ **ABCA orchestration complete**' + : kind === 'cancelled' + ? '🛑 **ABCA orchestration cancelled**' + : '⚠️ **ABCA orchestration finished with failures**'; + + const lines = [...children] + .sort((a, b) => (a.display_id ?? a.sub_issue_id).localeCompare(b.display_id ?? b.sub_issue_id)) + .map((c) => { + const icon = STATUS_ICON[c.child_status] ?? '•'; + const label = c.display_id + ? (c.title ? `${c.display_id}: ${c.title}` : c.display_id) + : (c.title ?? c.sub_issue_id); + // Append the child's PR link when one was opened, so the parent + // rollup is a single place to reach every sub-issue's PR. + const pr = c.pr_url ? ` — [PR](${c.pr_url})` : ''; + return `- ${icon} ${label} — ${c.child_status}${pr}`; + }); + + const summary = `${counts.succeeded} succeeded, ${counts.failed} failed, ${counts.skipped} skipped ` + + `(of ${children.length}).`; + + // Surface the integration node's combined PR as a prominent callout — + // it is the fan-out's single merged deliverable, and (being a synthetic node + // with no Linear sub-issue) it is otherwise unreachable from Linear. Only + // when the integration node actually opened a PR. + const integration = children.find((c) => isIntegrationNode(c.sub_issue_id) && c.pr_url); + const callout = integration + ? ['', `🔗 **Combined PR (all sub-issues merged):** [${integration.pr_url}](${integration.pr_url})`] + : []; + + return [heading, '', summary, ...callout, '', ...lines].join('\n'); +} + +/** + * Render the LIVE status block (pure) — the single edit-in-place comment on + * the parent epic that answers "where are we" during a running + * orchestration. Posted at seed and re-rendered + edited on + * every child transition, so the parent shows current progress without a + * comment stream. Once all children are terminal the reconciler replaces + * the body with the final {@link renderRollupComment}, so this block is the + * in-flight view only. + * + * Per-child line shows the same icons as the rollup (running/blocked/done/ + * failed/skipped) plus the child's PR link when known. + */ +export function renderStatusBlock(children: readonly RollupChildView[]): string { + const terminal = (s: string) => s === 'succeeded' || s === 'failed' || s === 'skipped'; + const done = children.filter((c) => terminal(c.child_status)).length; + + const heading = `🔄 **ABCA orchestration** · ${done}/${children.length} complete`; + + const lines = [...children] + .sort((a, b) => (a.display_id ?? a.sub_issue_id).localeCompare(b.display_id ?? b.sub_issue_id)) + .map((c) => { + const icon = STATUS_ICON[c.child_status] ?? '•'; + const label = c.display_id + ? (c.title ? `${c.display_id}: ${c.title}` : c.display_id) + : (c.title ?? c.sub_issue_id); + // Human-friendly status words for the in-flight view. 'releasing' (the + // transient flip-then-create claim) reads as running like + // released/ready — the child is mid-launch, not a distinct user-facing state. + const word = + c.child_status === 'released' || c.child_status === 'ready' || c.child_status === 'releasing' ? 'running' + : c.child_status === 'blocked' ? 'blocked' + : c.child_status; + // Link the PR as soon as it is known, even mid-run. + const pr = c.pr_url ? ` — [PR](${c.pr_url})` : ''; + return `- ${icon} ${label} — ${word}${pr}`; + }); + + return [heading, '', ...lines, '', '_Updates live as sub-issues progress._'].join('\n'); +} + +// ─────────────────────────────────────────────────────────────────────────── +// The single MATURING panel comment. Supersedes the +// separate renderStatusBlock + renderRollupComment — ONE comment, edited in +// place, that shows the full DAG and matures from in-progress → complete and +// back to in-progress on an extend/revision. +// ─────────────────────────────────────────────────────────────────────────── + +/** Per-sub-issue view for the maturing panel — adds the 'updating' context the rollup/block can't express. */ +export interface EpicPanelRow { + readonly sub_issue_id: string; + readonly display_id?: string; + readonly title?: string; + /** Persisted orchestration status: blocked | ready | released | succeeded | failed | skipped. */ + readonly child_status: string; + /** The sub-issue's current PR url, when one exists yet (omitted for a not-yet-PR'd first run). */ + readonly pr_url?: string; + /** + * When this row is being re-built by an in-flight cascade/iteration (its + * persisted status is still 'succeeded' but a new task is updating its PR), + * the human-readable reason — e.g. `per ENG-42's "button doesn't work"` or + * `to include ENG-42's change`. Present → the row renders as 🔄 updating. + */ + readonly updatingReason?: string; + /** + * SHORT one-line reason for a ❌ failed row, rendered as an indented sub-line: + * WHAT failed + WHERE to read it (CloudWatch by task + * id). Critical for the synthetic integration node, which has no Linear + * sub-issue / comment-iteration reply and would otherwise surface as a bare + * "❌ … — failed" with no diagnostic. Composed by + * {@link renderPanelFailureReason}; absent for non-failed rows. + */ + readonly failureReason?: string; +} + +export interface EpicPanelParams { + readonly rows: readonly EpicPanelRow[]; + /** + * True when any sub-issue is non-terminal OR any row is mid-update + * (cascade in flight). Drives the in-progress header even when every + * persisted status is terminal (a revision re-opens the epic). + */ + readonly inProgress: boolean; + /** Combined/integration PR url (the fan-out's merged deliverable), when one exists. */ + readonly combinedPrUrl?: string; + /** Combined preview screenshot url, embedded in the panel (auto-refreshes; no separate comment). */ + readonly combinedScreenshotUrl?: string; + /** + * Live deploy-preview URL the combined screenshot was captured from. + * When present, the embedded combined preview becomes a clickable + * deep-link to the running combined site. Ignored unless + * ``combinedScreenshotUrl`` is also set. + */ + readonly combinedPreviewUrl?: string; + /** + * The project's resolved trigger label, for the retry hint's fallback line. + * The label is per-project configurable, so this must NOT be hardcoded: a hint + * naming a label the webhook does not filter on sends the user to do something + * that silently does nothing. Absent → the platform default. + */ + readonly labelFilter?: string; +} + +const PANEL_FOOTER = '_One live panel — updates in place as the epic progresses; no comment stream._'; + +/** + * Truncate a quoted comment for the "updating per …" row, keeping it short. + * Exported so the caller (reconciler) builds the ``updatingReason`` string — + * e.g. ``per ENG-42's "${truncateQuote(commentBody)}"``. + */ +export function truncateQuote(s: string, max = 40): string { + const oneLine = s.replace(/\s+/g, ' ').trim(); + return oneLine.length <= max ? oneLine : `${oneLine.slice(0, max - 1)}…`; +} + +/** + * SHORT friendly name for a node, used where a node is NAMED inside prose (e.g. + * the cascade reason "updating to include 's change"). The integration node + * gets the friendly "the integration" rather than its raw stored title, so a + * possessive reads cleanly ("the integration's change") instead of leaking the + * clumsy synthetic title. Prefers the Linear identifier (ENG-42) for real + * nodes. (Observed in practice: the raw synthetic title read badly in prose.) + */ +export function cascadeNodeLabel( + subIssueId: string, + linearIdentifier?: string, + title?: string, +): string { + if (isIntegrationNode(subIssueId)) return 'the integration'; + return linearIdentifier ?? title ?? 'a predecessor'; +} + +/** Friendly label for a row — Linear identifier + title, or 'Integration — combined result' for the synthetic node. */ +function panelLabel(row: EpicPanelRow): string { + if (isIntegrationNode(row.sub_issue_id)) return 'Integration — combined result'; + if (row.display_id) return row.title ? `${row.display_id}: ${row.title}` : row.display_id; + return row.title ?? row.sub_issue_id; +} + +/** + * Render the single maturing epic panel (pure). Edited in place on every event + * (seed/run/extend/revision/complete). Rules: + * - PR link shown ONLY when a PR exists (a first run mid-flight has none). + * - A row with ``updatingReason`` renders as `🔄 … — updating — [PR]` + * even though its persisted status is still succeeded. + * - Header: in-progress → `🔄 N/M complete`; all settled → `✅ complete` or + * `⚠️ finished with failures`. ``inProgress`` forces 🔄 (a revision re-opens). + * - Integration node renders friendly; never a raw id. + * - Combined PR callout + embedded combined screenshot when present. + */ +export function renderEpicPanel(params: EpicPanelParams): string { + const { rows, inProgress, combinedPrUrl, combinedScreenshotUrl, combinedPreviewUrl } = params; + const triggerLabel = params.labelFilter?.trim() || DEFAULT_LABEL_FILTER; + const terminal = (s: string) => s === 'succeeded' || s === 'failed' || s === 'skipped'; + // "done" counts settled rows that are NOT mid-update (an updating row is back in flight). + const done = rows.filter((r) => terminal(r.child_status) && !r.updatingReason).length; + const anyBad = rows.some((r) => r.child_status === 'failed' || r.child_status === 'skipped'); + + let heading: string; + if (inProgress) { + heading = `🔄 **ABCA orchestration** · ${done}/${rows.length} complete`; + } else if (anyBad) { + heading = '⚠️ **ABCA orchestration finished with failures**'; + } else { + heading = '✅ **ABCA orchestration complete**'; + } + + const lines = [...rows] + .sort((a, b) => (a.display_id ?? a.sub_issue_id).localeCompare(b.display_id ?? b.sub_issue_id)) + .map((r) => { + const label = panelLabel(r); + const pr = r.pr_url ? ` — [PR](${r.pr_url})` : ''; + // A mid-update row: 🔄 + the reason, regardless of persisted status. + if (r.updatingReason) { + return `- 🔄 ${label} — updating ${r.updatingReason}${pr}`; + } + const icon = STATUS_ICON[r.child_status] ?? '•'; + const word = + r.child_status === 'released' || r.child_status === 'ready' || r.child_status === 'releasing' ? 'running' + : r.child_status === 'blocked' ? 'blocked' + : r.child_status; + const line = `- ${icon} ${label} — ${word}${pr}`; + // A failed row carries a diagnostic sub-line (what failed + the + // CloudWatch task to read). Indented continuation so it reads as a + // detail of the row, not a sibling bullet. Only when a reason was + // resolved (the integration node is the prime beneficiary — no sub-issue + // comment carries this anywhere else). + if (r.child_status === 'failed' && r.failureReason) { + return `${line}\n ↳ ${r.failureReason}`; + } + return line; + }); + + // A SETTLED epic that finished with failures tells the user how to retry. Names + // ONE way, deliberately: re-applying the label is a different gesture that only + // happens to retry in this one state — the same label re-apply also means "add + // the sub-issues I just created", "nothing to do, still running", or "already + // complete", inferred from the graph rather than from what the user meant. So an + // epic with both a new sub-issue and a failure does two things on a re-label and + // one on a retry, which is why the previous "either way" wording was wrong. + // Commenting is also the more reliable of the two: a label change only triggers + // when the webhook reports the label set as having changed, whereas a comment is + // an unambiguous event. The label is offered as a fallback, not an equal. + // Only when the epic is terminal (not inProgress) AND something failed/skipped + // (nothing to retry otherwise). One line so the panel stays scannable. + const retryHint = (!inProgress && anyBad) + ? ['', '↻ **To retry:** reply `@bgagent retry` on this epic — it re-runs only the ' + + 'failed/skipped sub-issues and keeps the ones that succeeded. ' + + `(No reply? Removing and re-applying the \`${triggerLabel}\` label also retries.)`] + : []; + + const callout = combinedPrUrl + ? ['', `🔗 **Combined PR (all sub-issues merged):** [${combinedPrUrl}](${combinedPrUrl})`] + : []; + // When we know the live preview-deploy URL, render the embedded + // screenshot as a clickable linked image + a plain "Open the combined + // preview" link, so a reviewer can open the running combined site, not just + // see a static PNG. The preview URL is payload-derived (came from the deploy + // webhook) — percent-encode its parens so a crafted path can't break out of + // the markdown link. The CloudFront screenshot URL is our own key (no + // parens) so it's interpolated as-is. + // + // "Combined" only when an integration node merged several leaves. On a chain the + // preview comes from the final node, and calling that "combined" would tell the + // reviewer several branches were merged when none were. + const previewLabel = combinedPrUrl ? 'Combined preview' : 'Preview'; + let shot: string[] = []; + if (combinedScreenshotUrl) { + if (combinedPreviewUrl) { + const safePreview = encodeMarkdownUrl(combinedPreviewUrl); + shot = [ + '', + `🖼️ **${previewLabel}**`, + '', + `[![${previewLabel.toLowerCase()}](${combinedScreenshotUrl})](${safePreview})`, + '', + `[Open the ${previewLabel.toLowerCase()}](${safePreview})`, + ]; + } else { + shot = ['', `🖼️ **${previewLabel}**`, '', `![${previewLabel.toLowerCase()}](${combinedScreenshotUrl})`]; + } + } + + return [heading, '', ...lines, ...callout, ...retryHint, ...shot, '', PANEL_FOOTER].join('\n'); +} + +/** + * Decide the rollup kind from the (terminal) child statuses. + * - any failed/skipped → partial_failure + * - all succeeded → complete + * (cancelled is passed explicitly by the cancel path, not derived here) + */ +export function rollupKindFromChildren(children: readonly RollupChildView[]): RollupKind { + const anyBad = children.some((c) => c.child_status === 'failed' || c.child_status === 'skipped'); + return anyBad ? 'partial_failure' : 'complete'; +} + +/** + * Build the {@link EpicPanelRow}s for a snapshot's children. Maps + * the persisted child rows + a ``sub_issue_id → pr_url`` map + an optional + * ``sub_issue_id → updatingReason`` map (rows a cascade is rebuilding) into the + * panel view. Pure. + */ +export function buildPanelRows( + children: readonly OrchestrationChildRow[], + prUrls: Readonly> = {}, + updating: Readonly> = {}, + failureReasons: Readonly> = {}, +): EpicPanelRow[] { + return children.map((c) => ({ + sub_issue_id: c.sub_issue_id, + ...(c.display_id !== undefined && { display_id: c.display_id }), + ...(c.title !== undefined && { title: c.title }), + child_status: c.child_status, + ...(prUrls[c.sub_issue_id] !== undefined && { pr_url: prUrls[c.sub_issue_id] }), + ...(updating[c.sub_issue_id] !== undefined && { updatingReason: updating[c.sub_issue_id] }), + ...(failureReasons[c.sub_issue_id] !== undefined && { failureReason: failureReasons[c.sub_issue_id] }), + })); +} + +export interface UpsertEpicPanelParams { + /** The surface adapter to drive. Optional capabilities it omits are skipped. */ + readonly channel: Channel; + /** The parent epic, on whichever surface triggered the orchestration. */ + readonly parent: IssueRef; + /** Existing panel comment id (status_comment_id). When absent, a fresh comment is posted + the id returned. */ + readonly statusCommentId?: string; + readonly children: readonly OrchestrationChildRow[]; + readonly prUrls?: Readonly>; + /** sub_issue_id → human reason, for rows a cascade is currently rebuilding. */ + readonly updating?: Readonly>; + /** + * sub_issue_id → one-line failure reason for a ❌ row. Resolved by the + * reconciler from the failed child task's record (build-gate vs agent-crash + + * CloudWatch task id). The integration node's entry is the one that matters + * most — it's the only place its combined-build failure can be surfaced. + */ + readonly failureReasons?: Readonly>; + readonly combinedPrUrl?: string; + readonly combinedScreenshotUrl?: string; + /** Live preview-deploy URL the combined screenshot was captured from. */ + readonly combinedPreviewUrl?: string; + /** + * Whether the epic is in progress. When omitted, derived: in progress iff any + * child is non-terminal OR any row has an updating reason. Pass explicitly to + * force (e.g. a revision just started → still in progress even if all + * persisted statuses are terminal). + */ + readonly inProgress?: boolean; + /** + * When true AND the epic is settled, mirror the outcome on the PARENT issue: + * advance state to awaiting-review (clean) / leave it (failures) + swap the + * reaction to ✅/❌. When in progress, revert: state → running + reaction → 👀. + * Skipped on surfaces without reaction/transition support. Default true. + */ + readonly mirrorParentState?: boolean; + /** + * The project's resolved trigger label, forwarded to {@link renderEpicPanel} + * for the retry hint. Absent → the platform default. Pass the project + * mapping's ``label_filter`` when the caller has it, so the hint never names a + * label that project renamed. + */ + readonly labelFilter?: string; +} + +/** + * Render + upsert the single maturing epic panel, and (optionally) mirror the + * outcome on the parent issue's state + reaction. The ONE place the parent panel + * is written. Returns the panel comment id (new or existing), or null on failure. + * + * - Edits ``statusCommentId`` in place when given; else posts a fresh comment. + * - Header/rows via {@link renderEpicPanel}; ``inProgress`` derived if omitted. + * - On settle (not in progress): advance parent state → awaiting review (clean) + * + ✅; on failures, leave the state and let ❌ convey it. On in-progress (a + * revision re-opened it): back to running + 👀. + * - Sequential, not concurrent: each mirror step fans out into several surface + * reads, and firing them together self-throttled the request budget so a + * transition silently no-op'd and left the epic stuck. + * Best-effort: a surface hiccup never throws out of the reconcile. + */ +export async function upsertEpicPanel(params: UpsertEpicPanelParams): Promise { + const { channel, parent } = params; + const rows = buildPanelRows(params.children, params.prUrls ?? {}, params.updating ?? {}, params.failureReasons ?? {}); + const terminal = (s: string) => s === 'succeeded' || s === 'failed' || s === 'skipped'; + const inProgress = params.inProgress + ?? rows.some((r) => !terminal(r.child_status) || r.updatingReason !== undefined); + const body = renderEpicPanel({ + rows, + inProgress, + ...(params.combinedPrUrl !== undefined && { combinedPrUrl: params.combinedPrUrl }), + ...(params.combinedScreenshotUrl !== undefined && { combinedScreenshotUrl: params.combinedScreenshotUrl }), + ...(params.combinedPreviewUrl !== undefined && { combinedPreviewUrl: params.combinedPreviewUrl }), + ...(params.labelFilter !== undefined && { labelFilter: params.labelFilter }), + }); + + let commentId: string | null; + try { + const ref = await channel.upsertComment( + parent, + body, + params.statusCommentId ? { commentId: params.statusCommentId } : undefined, + ); + // A surface that can't hand back a real comment id reports an empty one. + // Treat that as "no panel id" rather than persisting a blank id the next + // edit would try (and fail) to address. + commentId = ref?.commentId || null; + } catch (err) { + logger.warn('Epic panel upsert threw (non-fatal)', { + parent_issue_id: parent.issueId, + error: err instanceof Error ? err.message : String(err), + }); + return null; + } + + // Mirror parent state + reaction, sequentially (see the note above). + if (params.mirrorParentState !== false) { + const anyBad = rows.some((r) => r.child_status === 'failed' || r.child_status === 'skipped'); + try { + if (inProgress) { + // Re-opened (or running): back to running + 👀. This is a deliberate + // move backward within the same state category — a settled epic sits in + // "awaiting review", and re-opening it must be allowed explicitly or the + // adapter's backward-move guard drops it silently. A parent a human + // already marked done stays done; that guard still applies. + await channel.transitionState?.(parent, 'started', { allowRegression: true }); + await channel.replaceIssueReaction?.(parent, 'started'); + } else if (!anyBad) { + // Clean completion: work done, awaiting human merge → in review + ✅. + await channel.transitionState?.(parent, 'in_review'); + await channel.replaceIssueReaction?.(parent, 'succeeded'); + } else { + // Finished with failures: leave the state; the ❌ reaction conveys it. + await channel.replaceIssueReaction?.(parent, 'failed'); + } + } catch (err) { + logger.warn('Epic panel parent-state mirror failed (non-fatal)', { + parent_issue_id: parent.issueId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + return commentId; +} + +export interface PostRollupParams { + /** The surface adapter to drive. Optional capabilities it omits are skipped. */ + readonly channel: Channel; + readonly orchestrationId: string; + /** The parent epic, on whichever surface triggered the orchestration. */ + readonly parent: IssueRef; + readonly kind: RollupKind; + readonly children: readonly OrchestrationChildRow[]; + /** + * The live status-block comment id stamped at seed. When set, the + * final rollup EDITS that comment in place (one comment for the whole run, + * no stream). When absent (seed-time create failed, or an older + * orchestration), the rollup posts a fresh comment. + */ + readonly statusCommentId?: string; + /** + * ``sub_issue_id → pr_url`` for children that opened a PR. Supplied + * by the reconciler (batch-read from the TaskTable at rollup time, when + * pr_urls have settled). Threaded into the rendered comment as per-child + * links + the integration node's combined-PR callout. Absent/partial is + * fine — a missing entry just renders no link. + */ + readonly prUrls?: Readonly>; +} + +/** + * Post the parent rollup comment. Best-effort: never throws; logs a + * stable event on both success and failure so automated tests can assert + * on ``orch.rollup.posted`` / ``orch.rollup.failed``. + */ +export async function postRollup(params: PostRollupParams): Promise { + const { channel, parent, orchestrationId, kind, children, statusCommentId } = params; + const prUrls = params.prUrls ?? {}; + const body = renderRollupComment( + kind, + children.map((c) => ({ + sub_issue_id: c.sub_issue_id, + ...(c.display_id !== undefined && { display_id: c.display_id }), + ...(c.title !== undefined && { title: c.title }), + child_status: c.child_status, + ...(c.child_task_id !== undefined && { child_task_id: c.child_task_id }), + ...(prUrls[c.sub_issue_id] !== undefined && { pr_url: prUrls[c.sub_issue_id] }), + })), + ); + + let ok = false; + try { + // Edit the live status block into the final rollup when we have its id (one + // comment for the whole run); else post a fresh comment. + if (statusCommentId) { + ok = (await channel.upsertComment(parent, body, { commentId: statusCommentId })) !== null; + } else { + ok = (await channel.postComment(parent, body)) !== null; + } + } catch (err) { + logger.warn('Parent rollup comment threw (non-fatal)', { + event: ORCH_LOG.rollupFailed, + orchestration_id: orchestrationId, + parent_issue_id: parent.issueId, + rollup_kind: kind, + error: err instanceof Error ? err.message : String(err), + }); + return false; + } + + if (ok) { + logger.info('Parent rollup comment posted', { + event: ORCH_LOG.rollupPosted, + orchestration_id: orchestrationId, + parent_issue_id: parent.issueId, + rollup_kind: kind, + child_count: children.length, + }); + + // Mirror the children's status signal on the PARENT epic: + // - state: on a clean 'complete', advance to awaiting-review (work done, + // child PRs awaiting a human merge — NOT done, since nothing is merged). + // On a partial-failure / cancelled rollup, leave the state in place (the + // comment + ❌ reaction already convey the outcome). + // - reaction: replace the seed 👀 with ✅ (complete) / ❌ (otherwise) so the + // parent shows exactly ONE marker at a time, like the children. + // Run SEQUENTIALLY, not concurrently: the state transition and the reaction + // replace each fan out into multiple surface calls. Firing them together — + // on top of the comment edit just above — self-throttled the request budget, + // so the states read aborted and the transition silently no-op'd, leaving + // the parent stuck. Serialising keeps each read under its own budget. Both + // best-effort; a hiccup never suppresses the rollup. + if (kind === 'complete') { + await channel.transitionState?.(parent, 'in_review'); + } + await channel.replaceIssueReaction?.(parent, kind === 'complete' ? 'succeeded' : 'failed'); + } else { + logger.warn('Parent rollup comment post returned false', { + event: ORCH_LOG.rollupFailed, + orchestration_id: orchestrationId, + parent_issue_id: parent.issueId, + rollup_kind: kind, + }); + } + return ok; +} diff --git a/cdk/src/handlers/shared/orchestration-store.ts b/cdk/src/handlers/shared/orchestration-store.ts index 5e671191d..1c6f9a917 100644 --- a/cdk/src/handlers/shared/orchestration-store.ts +++ b/cdk/src/handlers/shared/orchestration-store.ts @@ -174,6 +174,37 @@ export interface OrchestrationReleaseContext { readonly linear_oauth_secret_arn?: string; readonly linear_workspace_slug?: string; readonly linear_project_id?: string; + /** + * The project's resolved trigger label (the project mapping's ``label_filter``). + * Persisted at SEED time because that is the only point where the mapping is in + * hand — the reconciler works from this row and has no project id to look one + * up with. Used for the epic panel's retry hint, which must name the label that + * actually fires: it is per-project configurable, so a hardcoded or defaulted + * one sends the user to re-apply something the webhook does not filter on. + * Absent on rows seeded before this field existed → the panel falls back to the + * platform default. + */ + readonly trigger_label?: string; + /** + * The parent epic's own title and body, captured at seed time and given to + * EVERY child as shared context. + * + * This is the fix for a real incident: an epic fanned out an API child and a UI + * child in parallel, each saw only its own sub-issue text, and each invented its + * own contract — one used `kyoto` with `checkIn`/`checkOut`, the other + * `wander-kyoto` with `startDate`/`endDate`. Both passed their own tests, the + * integration merged without conflict, and the deployed flow was broken. Nothing + * was wrong with either child in isolation; the shared agreement only ever + * existed in the parent, which neither of them could see. + * + * Seed time is the only place this is in hand — the reconciler works from the + * stored row and has no token to re-fetch the issue. Absent on rows seeded + * before this field existed, in which case children behave exactly as before. + */ + readonly parent_context?: { + readonly title?: string; + readonly description?: string; + }; /** * Parent-issue attachments, screened + stored ONCE at seed time, so every * child inherits them: the parent's attached spec must reach the agents that @@ -185,6 +216,28 @@ export interface OrchestrationReleaseContext { readonly pre_screened_attachments?: readonly AttachmentRecord[]; } +/** + * Per-field cap on the parent context copied onto the meta row. + * + * A bound is required for two independent reasons, and the smaller of the two + * wins: a DynamoDB item is capped at 400 KB total (this row also holds the + * release context and attachment JSON), and the text is prepended to EVERY + * child's task description, which is guardrail-screened and counts against the + * agent's prompt budget. 4000 characters is ample for an epic's shared contract + * while leaving both limits comfortable. + * + * Truncation is visible rather than silent: a reader of the child prompt sees + * that the text was cut, instead of quietly working from half a spec. + */ +export const PARENT_CONTEXT_MAX_CHARS = 4000; + +/** Cap one parent-context field, marking the cut so it is never mistaken for the whole. */ +function clampParentContext(text: string): string { + const trimmed = text.trim(); + if (trimmed.length <= PARENT_CONTEXT_MAX_CHARS) return trimmed; + return `${trimmed.slice(0, PARENT_CONTEXT_MAX_CHARS)}\n… [truncated]`; +} + export interface SeedOrchestrationParams { readonly ddb: DynamoDBDocumentClient; readonly tableName: string; @@ -421,6 +474,19 @@ export async function seedOrchestration( ...(releaseContext.linear_project_id !== undefined && { linear_project_id: releaseContext.linear_project_id, }), + ...(releaseContext.trigger_label !== undefined && { + trigger_label: releaseContext.trigger_label, + }), + // The epic's own title/body, inherited by every child so parallel siblings + // share one source of truth for names and shapes. Stored FLAT (not nested) + // and TRUNCATED here, at the single write site, so no reader has to remember + // to bound it — see PARENT_CONTEXT_MAX_CHARS for why a bound is required. + ...(releaseContext.parent_context?.title !== undefined && { + parent_context_title: clampParentContext(releaseContext.parent_context.title), + }), + ...(releaseContext.parent_context?.description !== undefined && { + parent_context_description: clampParentContext(releaseContext.parent_context.description), + }), // Parent attachments, inherited by every child — stored as a JSON string so the nested // AttachmentRecord[] round-trips cleanly through the Document client without // per-field marshalling. Omitted when the parent had none. @@ -902,6 +968,20 @@ export async function loadOrchestration( ...(metaItem.linear_project_id !== undefined && { linear_project_id: metaItem.linear_project_id as string, }), + ...(metaItem.trigger_label !== undefined && { + trigger_label: metaItem.trigger_label as string, + }), + ...((metaItem.parent_context_title !== undefined + || metaItem.parent_context_description !== undefined) && { + parent_context: { + ...(metaItem.parent_context_title !== undefined && { + title: metaItem.parent_context_title as string, + }), + ...(metaItem.parent_context_description !== undefined && { + description: metaItem.parent_context_description as string, + }), + }, + }), ...(preScreened.length > 0 && { pre_screened_attachments: preScreened }), }, ...(metaItem.retry_comment_id !== undefined && { diff --git a/cdk/test/constructs/orchestration-reconciler.test.ts b/cdk/test/constructs/orchestration-reconciler.test.ts new file mode 100644 index 000000000..49bc2967e --- /dev/null +++ b/cdk/test/constructs/orchestration-reconciler.test.ts @@ -0,0 +1,130 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { App, Stack } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import { OrchestrationReconciler } from '../../src/constructs/orchestration-reconciler'; +import { OrchestrationTable } from '../../src/constructs/orchestration-table'; +import { TaskEventsTable } from '../../src/constructs/task-events-table'; +import { TaskTable } from '../../src/constructs/task-table'; + +function synth(): Template { + const app = new App(); + const stack = new Stack(app, 'TestStack'); + const taskTable = new TaskTable(stack, 'TaskTable'); + const orchestrationTable = new OrchestrationTable(stack, 'OrchestrationTable'); + const taskEventsTable = new TaskEventsTable(stack, 'TaskEventsTable'); + new OrchestrationReconciler(stack, 'OrchestrationReconciler', { + taskTable: taskTable.table, + orchestrationTable: orchestrationTable.table, + taskEventsTable: taskEventsTable.table, + orchestratorFunctionArn: 'arn:aws:lambda:us-east-1:123456789012:function:orch', + }); + return Template.fromStack(stack); +} + +describe('OrchestrationReconciler', () => { + let template: Template; + beforeEach(() => { + template = synth(); + }); + + test('creates the reconciler Lambda with the orchestration table env', () => { + template.hasResourceProperties('AWS::Lambda::Function', { + Environment: { + Variables: Match.objectLike({ + ORCHESTRATION_TABLE_NAME: Match.anyValue(), + TASK_TABLE_NAME: Match.anyValue(), + }), + }, + }); + }); + + test('subscribes to the TaskTable stream via an event-source mapping', () => { + template.resourceCountIs('AWS::Lambda::EventSourceMapping', 1); + template.hasResourceProperties('AWS::Lambda::EventSourceMapping', { + StartingPosition: 'LATEST', + BisectBatchOnFunctionError: true, + }); + }); + + test('filters the stream to TERMINAL statuses only (skips RUNNING/heartbeat churn)', () => { + // The handler ignores non-terminal records; the stream FilterCriteria makes + // that explicit so every non-terminal TaskTable write platform-wide doesn't + // invoke the reconciler. One filter pattern per terminal status (OR-ed). + template.hasResourceProperties('AWS::Lambda::EventSourceMapping', { + FilterCriteria: { + Filters: Match.arrayWith([ + Match.objectLike({ + Pattern: Match.stringLikeRegexp('"status":\\{"S":\\["COMPLETED"\\]\\}'), + }), + Match.objectLike({ + Pattern: Match.stringLikeRegexp('"status":\\{"S":\\["FAILED"\\]\\}'), + }), + ]), + }, + }); + }); + + test('provisions a DLQ for poison stream records', () => { + // At least one SQS queue (the reconciler DLQ). + const queues = template.findResources('AWS::SQS::Queue'); + expect(Object.keys(queues).length).toBeGreaterThanOrEqual(1); + }); + + test('TaskTable has a stream enabled (reconciler source)', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + StreamSpecification: { StreamViewType: 'NEW_IMAGE' }, + }); + }); +}); + +describe('OrchestrationReconciler — grants', () => { + test('grants the function read/write on the orchestration table', () => { + const template = synth(); + // The function role should have a policy referencing dynamodb actions. + const policies = template.findResources('AWS::IAM::Policy'); + const hasDdb = Object.values(policies).some((p) => { + const statements = (p.Properties as { PolicyDocument: { Statement: Array<{ Action?: unknown }> } }) + .PolicyDocument.Statement; + return JSON.stringify(statements).includes('dynamodb:'); + }); + expect(hasDdb).toBe(true); + }); +}); + +// Minimal sanity that the props type accepts an ITable. +describe('OrchestrationReconciler — typing', () => { + test('accepts imported tables', () => { + const app = new App(); + const stack = new Stack(app, 'T2'); + const taskTable = dynamodb.Table.fromTableAttributes(stack, 'TT', { + tableName: 'tasks', + tableStreamArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/tasks/stream/2026', + }); + const orch = dynamodb.Table.fromTableName(stack, 'OT', 'orch'); + const events = dynamodb.Table.fromTableName(stack, 'ET', 'events'); + expect(() => new OrchestrationReconciler(stack, 'R', { + taskTable, + orchestrationTable: orch, + taskEventsTable: events, + })).not.toThrow(); + }); +}); diff --git a/cdk/test/constructs/task-table.test.ts b/cdk/test/constructs/task-table.test.ts index 638d289e3..0da53df26 100644 --- a/cdk/test/constructs/task-table.test.ts +++ b/cdk/test/constructs/task-table.test.ts @@ -104,6 +104,24 @@ describe('TaskTable', () => { }); }); + test('creates LinearIssueIndex GSI (PK linear_issue_id, SK created_at, INCLUDE projection)', () => { + template.hasResourceProperties('AWS::DynamoDB::Table', { + GlobalSecondaryIndexes: Match.arrayWith([ + Match.objectLike({ + IndexName: 'LinearIssueIndex', + KeySchema: [ + { AttributeName: 'linear_issue_id', KeyType: 'HASH' }, + { AttributeName: 'created_at', KeyType: 'RANGE' }, + ], + Projection: { + ProjectionType: 'INCLUDE', + NonKeyAttributes: Match.arrayWith(['pr_url', 'pr_number', 'status', 'repo', 'user_id', 'channel_metadata']), + }, + }), + ]), + }); + }); + test('creates sparse JiraIssueIndex GSI with the resolver projection', () => { template.hasResourceProperties('AWS::DynamoDB::Table', { GlobalSecondaryIndexes: Match.arrayWith([ @@ -138,6 +156,7 @@ describe('TaskTable', () => { { AttributeName: 'status', AttributeType: 'S' }, { AttributeName: 'created_at', AttributeType: 'S' }, { AttributeName: 'idempotency_key', AttributeType: 'S' }, + { AttributeName: 'linear_issue_id', AttributeType: 'S' }, { AttributeName: 'jira_issue_identity', AttributeType: 'S' }, ]), }); @@ -156,6 +175,7 @@ describe('TaskTable', () => { expect(TaskTable.USER_STATUS_INDEX).toBe('UserStatusIndex'); expect(TaskTable.STATUS_INDEX).toBe('StatusIndex'); expect(TaskTable.IDEMPOTENCY_INDEX).toBe('IdempotencyIndex'); + expect(TaskTable.LINEAR_ISSUE_INDEX).toBe('LinearIssueIndex'); expect(TaskTable.JIRA_ISSUE_INDEX).toBe('JiraIssueIndex'); expect(JIRA_ISSUE_INDEX_NAME).toBe(TaskTable.JIRA_ISSUE_INDEX); }); diff --git a/cdk/test/handlers/orchestration-reconciler.test.ts b/cdk/test/handlers/orchestration-reconciler.test.ts new file mode 100644 index 000000000..63d5433cb --- /dev/null +++ b/cdk/test/handlers/orchestration-reconciler.test.ts @@ -0,0 +1,1464 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import type { DynamoDBRecord } from 'aws-lambda'; + +const ddbSend = jest.fn(); +jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: ddbSend })) }, + QueryCommand: jest.fn((input: unknown) => ({ _type: 'Query', input })), + UpdateCommand: jest.fn((input: unknown) => ({ _type: 'Update', input })), + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), + BatchGetCommand: jest.fn((input: unknown) => ({ _type: 'BatchGet', input })), + PutCommand: jest.fn((input: unknown) => ({ _type: 'Put', input })), +})); + +const s3SendMock = jest.fn(); +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: s3SendMock })), + GetObjectCommand: jest.fn((input: unknown) => ({ _type: 'S3Get', input })), +})); + +const resolveLinearOauthTokenMock = jest.fn(); +jest.mock('../../src/handlers/shared/linear-oauth-resolver', () => ({ + resolveLinearOauthToken: (...args: unknown[]) => resolveLinearOauthTokenMock(...args), +})); + +const createTaskCoreMock = jest.fn(); +jest.mock('../../src/handlers/shared/create-task-core', () => ({ + createTaskCore: (...args: unknown[]) => createTaskCoreMock(...args), +})); + +const postIssueCommentMock = jest.fn(); +const upsertStatusCommentMock = jest.fn(); +const swapIssueReactionMock = jest.fn(); +const swapCommentReactionMock = jest.fn(); +const transitionIssueStateMock = jest.fn(); +const revertIssueToNotStartedMock = jest.fn(); +const replyToCommentMock = jest.fn(); +const upsertThreadedReplyMock = jest.fn(); +jest.mock('../../src/handlers/shared/linear-feedback', () => ({ + postIssueComment: (...args: unknown[]) => postIssueCommentMock(...args), + upsertStatusComment: (...args: unknown[]) => upsertStatusCommentMock(...args), + swapIssueReaction: (...args: unknown[]) => swapIssueReactionMock(...args), + swapCommentReaction: (...args: unknown[]) => swapCommentReactionMock(...args), + transitionIssueState: (...args: unknown[]) => transitionIssueStateMock(...args), + revertIssueToNotStarted: (...args: unknown[]) => revertIssueToNotStartedMock(...args), + replyToComment: (...args: unknown[]) => replyToCommentMock(...args), + upsertThreadedReply: (...args: unknown[]) => upsertThreadedReplyMock(...args), + EMOJI_SUCCESS: 'white_check_mark', + EMOJI_FAILURE: 'x', + EMOJI_NEEDS_INPUT: 'question', +})); + +jest.mock('../../src/handlers/shared/logger', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +process.env.ORCHESTRATION_TABLE_NAME = 'OrchestrationTable'; +process.env.TASK_TABLE_NAME = 'TaskTable'; +// Cascade surfacing: the cascade posts Linear comments only when the +// workspace registry is configured. Set it so the surfacing path is exercised. +process.env.LINEAR_WORKSPACE_REGISTRY_TABLE_NAME = 'WorkspaceRegistry'; +process.env.ARTIFACTS_BUCKET_NAME = 'ArtifactsBucket'; + +import { TERMINAL_STATUSES } from '../../src/constructs/task-status'; +import { handler, parseTerminalTaskRecord } from '../../src/handlers/orchestration-reconciler'; + +/** Build a TaskTable stream MODIFY record. */ +function taskRecord(fields: { + task_id?: string; + status?: string; + build_passed?: boolean; + orchestration_id?: string; + eventName?: 'INSERT' | 'MODIFY' | 'REMOVE'; + // Cascade markers (channel_metadata fields on an iteration/restack task). + orchestration_sub_issue_id?: string; + restack_predecessor_sub_issue_id?: string; + orchestration_iteration?: boolean; + // The human comment that triggered an iteration. + trigger_comment_id?: string; + // The issue that trigger comment lives on (the parent epic when routed). + trigger_comment_issue_id?: string; + // Raw agent error_message (drives the failure-reply detail). + error_message?: string; + // Whether the agent actually edited code; false marks a question/answer run. + code_changed?: boolean; + // Stream sequence number (itemIdentifier for partial-batch reporting). + sequenceNumber?: string; +}): DynamoDBRecord { + const img: Record = {}; + if (fields.task_id) img.task_id = { S: fields.task_id }; + if (fields.status) img.status = { S: fields.status }; + if (fields.build_passed !== undefined) img.build_passed = { BOOL: fields.build_passed }; + if (fields.code_changed !== undefined) img.code_changed = { BOOL: fields.code_changed }; + if (fields.error_message) img.error_message = { S: fields.error_message }; + // PRODUCTION SHAPE: createTaskCore persists orchestration_id INSIDE the + // nested channel_metadata MAP, not as a top-level attribute. The stream + // image must mirror that or the reconciler skips every orchestration + // child. (Regression: the first dev smoke had orchestration_id only in + // channel_metadata and the reconciler — reading it top-level — ignored + // all completions, so dependents never released.) + const cm: Record = {}; + if (fields.orchestration_id) cm.orchestration_id = { S: fields.orchestration_id }; + if (fields.orchestration_sub_issue_id) cm.orchestration_sub_issue_id = { S: fields.orchestration_sub_issue_id }; + if (fields.restack_predecessor_sub_issue_id) { + cm.restack_predecessor_sub_issue_id = { S: fields.restack_predecessor_sub_issue_id }; + } + if (fields.orchestration_iteration) cm.orchestration_iteration = { S: 'true' }; + if (fields.trigger_comment_id) cm.trigger_comment_id = { S: fields.trigger_comment_id }; + if (fields.trigger_comment_issue_id) cm.trigger_comment_issue_id = { S: fields.trigger_comment_issue_id }; + if (Object.keys(cm).length > 0) img.channel_metadata = { M: cm }; + return { + eventName: fields.eventName ?? 'MODIFY', + dynamodb: { + NewImage: img as never, + ...(fields.sequenceNumber !== undefined && { SequenceNumber: fields.sequenceNumber }), + }, + } as DynamoDBRecord; +} + +describe('parseTerminalTaskRecord', () => { + test('extracts a terminal orchestration child event', () => { + const evt = parseTerminalTaskRecord(taskRecord({ + task_id: 'T1', status: 'COMPLETED', build_passed: true, orchestration_id: 'orch_1', + })); + expect(evt).toEqual({ taskId: 'T1', status: 'COMPLETED', buildPassed: true, orchestrationId: 'orch_1' }); + }); + + test('skips non-terminal status', () => { + expect(parseTerminalTaskRecord(taskRecord({ task_id: 'T1', status: 'RUNNING', orchestration_id: 'orch_1' }))).toBeNull(); + }); + + test('skips tasks with no orchestration_id (non-orchestration tasks)', () => { + expect(parseTerminalTaskRecord(taskRecord({ task_id: 'T1', status: 'COMPLETED' }))).toBeNull(); + }); + + test('skips REMOVE events', () => { + expect(parseTerminalTaskRecord(taskRecord({ + task_id: 'T1', status: 'COMPLETED', orchestration_id: 'orch_1', eventName: 'REMOVE', + }))).toBeNull(); + }); + + test('skips records with no NewImage', () => { + expect(parseTerminalTaskRecord({ eventName: 'MODIFY', dynamodb: {} } as DynamoDBRecord)).toBeNull(); + }); + + test('skips a terminal task carrying no orchestration_id (not an orchestration child)', () => { + // The gate is the presence of orchestration_id, not the workflow: a task that + // is not part of a graph must fall through rather than be mis-gated as a child + // and released against a graph it has nothing to do with. + expect(parseTerminalTaskRecord(plainTerminalRecord({ task_id: 'P1', status: 'COMPLETED' }))).toBeNull(); + }); + + // Drift guard. The stream's FilterCriteria is built from TERMINAL_STATUSES, and + // this handler decides independently which arriving records it acts on. If the + // two ever disagree, a status either never reaches the handler or reaches it and + // is dropped — and both look exactly like nothing happening. Driving EVERY + // terminal status through the parser fails on drift in either direction: a + // status added to the filter but not honoured here, or dropped here while the + // filter still admits it. + test.each(TERMINAL_STATUSES)('acts on %s — every status the stream filter admits', (status) => { + const evt = parseTerminalTaskRecord(taskRecord({ + task_id: 'T1', status, orchestration_id: 'orch_1', + })); + expect(evt).not.toBeNull(); + expect(evt?.status).toBe(status); + }); + + test('a non-terminal status is NOT acted on, so the guard above cannot pass by accepting everything', () => { + for (const status of ['RUNNING', 'PENDING', 'AWAITING_APPROVAL']) { + expect(parseTerminalTaskRecord(taskRecord({ + task_id: 'T1', status, orchestration_id: 'orch_1', + }))).toBeNull(); + } + }); +}); + +/** Build a terminal task stream record that carries NO orchestration_id. */ +function plainTerminalRecord(fields: { task_id?: string; status?: string }): DynamoDBRecord { + return { + eventName: 'MODIFY', + dynamodb: { + NewImage: { + task_id: { S: fields.task_id ?? 'T1' }, + status: { S: fields.status ?? 'COMPLETED' }, + resolved_workflow: { M: { id: { S: 'coding/new-task-v1' }, version: { S: '1.0.0' } } }, + user_id: { S: 'user-1' }, + repo: { S: 'o/r' }, + } as never, + }, + } as DynamoDBRecord; +} + +/** Mock the GSI lookup + loadOrchestration Query for a child set. */ +function mockOrchestration(opts: { + subIssueId: string; + children: Array<{ sub_issue_id: string; depends_on?: string[]; child_status: string }>; + /** Extra meta-row attributes, e.g. the recorded ``channel_source``. */ + meta?: Record; +}): void { + // Stateful, query-type-aware mock (robust to the reconciler's read + // pattern: GSI lookup + possibly-repeated loadOrchestration + status + // Updates). Status Updates mutate the in-memory rows so a subsequent + // fresh loadOrchestration reflects them — which is exactly what the + // concurrency-safe re-read relies on. + const meta = { + sub_issue_id: '#meta', + orchestration_id: 'orch_1', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + child_count: opts.children.length, + platform_user_id: 'user-1', + ...opts.meta, + }; + const rows: Record> = {}; + for (const c of opts.children) { + rows[c.sub_issue_id] = { + orchestration_id: 'orch_1', + sub_issue_id: c.sub_issue_id, + depends_on: c.depends_on ?? [], + child_status: c.child_status, + repo: 'o/r', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + }; + } + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + const { _type, input } = cmd; + if (_type === 'Query' && input.IndexName === 'ChildTaskIndex') { + return { Items: [{ ...rows[opts.subIssueId], sub_issue_id: opts.subIssueId }] }; + } + if (_type === 'Query') { // loadOrchestration + return { Items: [meta, ...Object.values(rows)] }; + } + if (_type === 'Update') { + const sk = (input.Key as { sub_issue_id: string }).sub_issue_id; + const vals = input.ExpressionAttributeValues as Record; + const row = rows[sk]; + if (row) { + if (vals[':s'] !== undefined) row.child_status = vals[':s']; + if (vals[':released'] !== undefined) { row.child_status = 'released'; row.child_task_id = vals[':tid']; } + } + return {}; + } + return {}; + }); +} + +describe('orchestration-reconciler handler', () => { + beforeEach(() => { + ddbSend.mockReset(); + createTaskCoreMock.mockReset(); + createTaskCoreMock.mockResolvedValue({ statusCode: 201, body: JSON.stringify({ data: { task_id: 'child-task' } }) }); + }); + + test('A succeeds → releases blocked dependent B', async () => { + mockOrchestration({ + subIssueId: 'A', + children: [ + { sub_issue_id: 'A', child_status: 'released' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'blocked' }, + ], + }); + await handler({ Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1' })] } as never); + + // B released via createTaskCore. + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const ctx = createTaskCoreMock.mock.calls[0][1]; + expect(ctx.idempotencyKey).toBe('orch_1_B'); + }); + + test('A fails → no release, B skipped (createTaskCore not called)', async () => { + mockOrchestration({ + subIssueId: 'A', + children: [ + { sub_issue_id: 'A', child_status: 'released' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'blocked' }, + ], + }); + + await handler({ Records: [taskRecord({ task_id: 'TA', status: 'FAILED', orchestration_id: 'orch_1' })] } as never); + + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + // A child that fails DETERMINISTICALLY at release (guardrail 400) must not + // only terminally-fail itself but ALSO transitively skip its dependents — + // else a mid-graph guardrail failure hangs the epic at the blocked dependent + // instead of looping. Failing only the leaf is not enough; the transitive + // skip has to run too. + test('A succeeds → B released but its create is guardrail-blocked (400) → B failed + dependent C skipped', async () => { + // Graph: A (done) → B (about to release, will be guardrail-blocked) → C. + mockOrchestration({ + subIssueId: 'A', + children: [ + { sub_issue_id: 'A', child_status: 'succeeded' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'blocked' }, + { sub_issue_id: 'C', depends_on: ['B'], child_status: 'blocked' }, + ], + }); + // B's task creation is deterministically blocked (guardrail); every other + // create (there is none here) would 201. releaseReadyChildren releases B → + // createTaskCore(B) → 400 → create_failed_terminal. + createTaskCoreMock.mockReset().mockResolvedValue({ + statusCode: 400, + body: '{"error":{"message":"Task description was blocked by content policy."}}', + }); + + await handler({ Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1' })] } as never); + + // The reconciler tried to release B (one createTaskCore for the ready node). + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + // B was written terminally 'failed' (failClaimTerminal) AND C was written + // 'skipped' (transitive skip) — both via conditional Updates on the store. + const updates = ddbSend.mock.calls + .map((c) => c[0] as { _type: string; input: Record }) + .filter((c) => c._type === 'Update'); + const wrote = (sk: string, status: string) => updates.some((u) => + (u.input.Key as { sub_issue_id?: string }).sub_issue_id === sk + && (u.input.ExpressionAttributeValues as Record)[':s'] === status + || (sk === 'B' && (u.input.Key as { sub_issue_id?: string }).sub_issue_id === 'B' + && (u.input.ExpressionAttributeValues as Record)[':failed'] === 'failed')); + // B → failed (failClaimTerminal uses :failed), C → skipped (:s). + expect(updates.some((u) => + (u.input.Key as { sub_issue_id?: string }).sub_issue_id === 'B' + && (u.input.ExpressionAttributeValues as Record)[':failed'] === 'failed')).toBe(true); + expect(wrote('C', 'skipped')).toBe(true); + // The epic settled (panel posted) — not left hanging on a blocked C. + expect(upsertStatusCommentMock).toHaveBeenCalled(); + }); + + test('COMPLETED with build_passed=false → treated as failure, B not released', async () => { + mockOrchestration({ + subIssueId: 'A', + children: [ + { sub_issue_id: 'A', child_status: 'released' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'blocked' }, + ], + }); + + await handler({ + Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', build_passed: false, orchestration_id: 'orch_1' })], + } as never); + + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('a build-gate-failed child (COMPLETED, build_passed=false) reverts state AND swaps its ✅ reaction to ❌', async () => { + // The agent moves a writeable child to "In Review" + reacts ✅ on agent-success + // (regression-only build gate), but the platform gate independently marks it + // failed. Left alone, the graph says failed while Linear reads "In Review" with + // a ✅ reaction + PR link (the user's inconsistency). The reconciler pulls the + // child back to not-started AND settles the reaction ✅→❌. + revertIssueToNotStartedMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + mockOrchestration({ + subIssueId: 'A', + children: [{ sub_issue_id: 'A', child_status: 'released' }], + }); + await handler({ + Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', build_passed: false, orchestration_id: 'orch_1' })], + } as never); + expect(revertIssueToNotStartedMock).toHaveBeenCalledWith(expect.anything(), 'A'); + expect(swapIssueReactionMock).toHaveBeenCalledWith(expect.anything(), 'A', 'x'); + }); + + test('a genuinely FAILED child also reverts state + swaps reaction to ❌', async () => { + revertIssueToNotStartedMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + mockOrchestration({ + subIssueId: 'A', + children: [{ sub_issue_id: 'A', child_status: 'released' }], + }); + await handler({ + Records: [taskRecord({ task_id: 'TA', status: 'FAILED', orchestration_id: 'orch_1' })], + } as never); + expect(revertIssueToNotStartedMock).toHaveBeenCalledWith(expect.anything(), 'A'); + expect(swapIssueReactionMock).toHaveBeenCalledWith(expect.anything(), 'A', 'x'); + }); + + test('a SUCCEEDING child is never reverted or ❌-reacted (leaves ✅ + In Review intact)', async () => { + revertIssueToNotStartedMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + mockOrchestration({ + subIssueId: 'A', + children: [{ sub_issue_id: 'A', child_status: 'released' }], + }); + await handler({ + Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1' })], + } as never); + expect(revertIssueToNotStartedMock).not.toHaveBeenCalledWith(expect.anything(), 'A'); + expect(swapIssueReactionMock).not.toHaveBeenCalledWith(expect.anything(), 'A', 'x'); + }); + + test('non-orchestration / non-terminal records are skipped entirely', async () => { + await handler({ + Records: [ + taskRecord({ task_id: 'T1', status: 'RUNNING', orchestration_id: 'orch_1' }), + taskRecord({ task_id: 'T2', status: 'COMPLETED' }), // no orchestration_id + ], + } as never); + expect(ddbSend).not.toHaveBeenCalled(); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('unresolvable sub_issue_id (GSI miss) → skip, no throw', async () => { + ddbSend.mockResolvedValueOnce({ Items: [] }); // GSI miss + await handler({ Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1' })] } as never); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('an all-terminal epic with an integration node → embeds its combined screenshot in the panel', async () => { + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-1'); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + const meta = { + sub_issue_id: '#meta', + orchestration_id: 'orch_1', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + child_count: 2, + platform_user_id: 'u1', + status_comment_id: 'panel-1', + }; + // A (real leaf) + integration node, BOTH succeeded → all-terminal. The + // integration node's task record carries a screenshot_url. + const rows = [ + { + orchestration_id: 'orch_1', + sub_issue_id: 'A', + depends_on: [], + child_status: 'succeeded', + child_task_id: 'task-A', + repo: 'o/r', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + linear_identifier: 'ENG-1', + }, + { + orchestration_id: 'orch_1', + sub_issue_id: 'orch_1__integration', + depends_on: ['A'], + child_status: 'succeeded', + child_task_id: 'task-int', + repo: 'o/r', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + }, + ]; + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + if (cmd._type === 'Query' && cmd.input.IndexName === 'ChildTaskIndex') { + return { Items: [{ ...rows[1] }] }; // the integration node just completed + } + if (cmd._type === 'Query') return { Items: [meta, ...rows] }; + if (cmd._type === 'BatchGet') { // resolveChildPrUrls + const keys = cmd.input.RequestItems as Record }>; + const tbl = Object.keys(keys)[0]; + return { Responses: { [tbl]: keys[tbl].Keys.map((k) => ({ task_id: k.task_id, pr_url: `https://github.com/o/r/pull/${k.task_id.length}` })) } }; + } + if (cmd._type === 'Get') { // resolveCombinedScreenshotUrl(task-int) + const tid = (cmd.input.Key as { task_id: string }).task_id; + return { + Item: tid === 'task-int' + ? { screenshot_url: 'https://cdn.example/combined.png', screenshot_preview_url: 'https://combined.vercel.app' } + : {}, + }; + } + return {}; + }); + + await handler({ + Records: [taskRecord({ + task_id: 'task-int', status: 'COMPLETED', orchestration_id: 'orch_1', + })], + } as never); + + expect(upsertStatusCommentMock).toHaveBeenCalled(); + const body = upsertStatusCommentMock.mock.calls.at(-1)![2] as string; + expect(body).toContain('✅'); // complete + // The panel embeds the image AND deep-links to the live combined deploy. + expect(body).toContain('[![combined preview](https://cdn.example/combined.png)](https://combined.vercel.app)'); + expect(body).toContain('[Open the combined preview](https://combined.vercel.app)'); + }); + + test('a FAILED integration node surfaces its build-failure reason + CloudWatch pointer on the panel', async () => { + // the synthetic integration node has no Linear sub-issue, + // so a failed combined build previously surfaced as a bare "❌ … failed" with + // NO reason and NO log pointer. The reconciler must now resolve the reason + // from the failed task's record and render it as a panel sub-line. + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-1'); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + const meta = { + sub_issue_id: '#meta', + orchestration_id: 'orch_1', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + child_count: 2, + platform_user_id: 'u1', + status_comment_id: 'panel-1', + }; + // A succeeded leaf + a FAILED integration node → all-terminal (with failures). + const rows = [ + { + orchestration_id: 'orch_1', + sub_issue_id: 'A', + depends_on: [], + child_status: 'succeeded', + child_task_id: 'task-A', + repo: 'o/r', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + linear_identifier: 'ENG-1', + }, + { + orchestration_id: 'orch_1', + sub_issue_id: 'orch_1__integration', + depends_on: ['A'], + child_status: 'failed', + child_task_id: 'task-int', + repo: 'o/r', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + }, + ]; + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + if (cmd._type === 'Query' && cmd.input.IndexName === 'ChildTaskIndex') { + return { Items: [{ ...rows[1] }] }; // the integration node just went terminal (failed) + } + if (cmd._type === 'Query') return { Items: [meta, ...rows] }; + if (cmd._type === 'BatchGet') { + const keys = cmd.input.RequestItems as Record; ProjectionExpression?: string }>; + const tbl = Object.keys(keys)[0]; + const proj = keys[tbl].ProjectionExpression ?? ''; + // resolveChildFailureReasons projects error_message/build_passed; the + // failed integration task carries the real build-gate error shape. + if (proj.includes('error_message')) { + return { + Responses: { + [tbl]: keys[tbl].Keys.map((k) => ( + k.task_id === 'task-int' + ? { task_id: k.task_id, error_message: "Task did not succeed (agent_status='success', build_ok=False)" } + : { task_id: k.task_id } + )), + }, + }; + } + // resolveChildPrUrls projects task_id/pr_url. + return { Responses: { [tbl]: keys[tbl].Keys.map((k) => ({ task_id: k.task_id, pr_url: `https://github.com/o/r/pull/${k.task_id.length}` })) } }; + } + return {}; + }); + + await handler({ + Records: [taskRecord({ task_id: 'task-int', status: 'FAILED', orchestration_id: 'orch_1' })], + } as never); + + expect(upsertStatusCommentMock).toHaveBeenCalled(); + const body = upsertStatusCommentMock.mock.calls.at(-1)![2] as string; + expect(body).toContain('⚠️ **ABCA orchestration finished with failures**'); + // The diagnostic sub-line: names the combined merge build + points at CloudWatch by task id. + expect(body).toMatch(/↳ Combined build failed after merging the sub-issue branches/); + expect(body).toContain('CloudWatch for task `task-int`'); + // Never leaks raw build output (untrusted repo content). + expect(body).not.toContain('build_ok'); + }); + + // Partial-batch failure reporting. A record whose processing throws must be + // reported by its sequence number (so only IT retries), not throw out of the + // handler (which fails + re-drives the whole batch, re-reconciling healthy + // siblings). + test('a record that throws is reported in batchItemFailures, not rethrown; a healthy sibling still commits', async () => { + // Make every DDB read throw so processing the FIRST record errors. The + // second record (no sequence number) is processed after — with the throwing + // mock it also errors, but has no seq, so it would rethrow; give BOTH a seq. + ddbSend.mockRejectedValue(new Error('DDB throttled')); + const rec1 = taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1', sequenceNumber: 'seq-1' }); + const rec2 = taskRecord({ task_id: 'TB', status: 'COMPLETED', orchestration_id: 'orch_2', sequenceNumber: 'seq-2' }); + + const res = (await handler({ Records: [rec1, rec2] } as never)) as { batchItemFailures: { itemIdentifier: string }[] }; + // Both failed → both reported; the handler did NOT throw. + expect(res.batchItemFailures.map((f) => f.itemIdentifier).sort()).toEqual(['seq-1', 'seq-2']); + }); + + test('a happy batch returns an empty batchItemFailures (nothing retried)', async () => { + mockOrchestration({ + subIssueId: 'A', + children: [ + { sub_issue_id: 'A', child_status: 'released' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'blocked' }, + ], + }); + const res = (await handler({ + Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1', sequenceNumber: 'seq-1' })], + } as never)) as { batchItemFailures: { itemIdentifier: string }[] }; + expect(res.batchItemFailures).toEqual([]); + }); + + test('a throwing record with NO sequence number rethrows (cannot isolate — fail the batch, never silently drop)', async () => { + ddbSend.mockRejectedValue(new Error('DDB throttled')); + const rec = taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1' }); // no sequenceNumber + await expect(handler({ Records: [rec] } as never)).rejects.toThrow('DDB throttled'); + }); +}); + +/** Detect a cascade marker in parseTerminalTaskRecord. */ +describe('parseTerminalTaskRecord — the cascade marker', () => { + test('a restack task (carries restack_predecessor) → cascadeSubIssueId set', () => { + const evt = parseTerminalTaskRecord(taskRecord({ + task_id: 'TR', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })); + expect(evt?.cascadeSubIssueId).toBe('B'); + }); + + test('an iteration task (orchestration_iteration=true) → cascadeSubIssueId set', () => { + const evt = parseTerminalTaskRecord(taskRecord({ + task_id: 'TI', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + })); + expect(evt?.cascadeSubIssueId).toBe('A'); + }); + + test('a normal child task (no markers) → cascadeSubIssueId undefined', () => { + const evt = parseTerminalTaskRecord(taskRecord({ + task_id: 'T1', status: 'COMPLETED', orchestration_id: 'orch_1', + })); + expect(evt?.cascadeSubIssueId).toBeUndefined(); + }); +}); + +/** Mock for the cascade path: loadOrchestration + per-dependent GetCommand pr_url. */ +function mockCascade(children: Array<{ + sub_issue_id: string; + depends_on?: string[]; + child_status: string; + child_task_id?: string; + child_branch_name?: string; + linear_identifier?: string; +}>, metaOverrides: Record = {}): void { + const meta = { + sub_issue_id: '#meta', + orchestration_id: 'orch_1', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + child_count: children.length, + platform_user_id: 'user-1', + // A panel comment exists → the cascade EDITS it, rather than posting fresh. + status_comment_id: 'panel-cmt-1', + ...metaOverrides, + }; + const rows = children.map((c) => ({ + orchestration_id: 'orch_1', + sub_issue_id: c.sub_issue_id, + depends_on: c.depends_on ?? [], + child_status: c.child_status, + repo: 'o/r', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + ...(c.child_task_id && { child_task_id: c.child_task_id }), + ...(c.child_branch_name && { child_branch_name: c.child_branch_name }), + ...(c.linear_identifier && { linear_identifier: c.linear_identifier }), + })); + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + if (cmd._type === 'Query') return { Items: [meta, ...rows] }; // loadOrchestration + if (cmd._type === 'Get') { // resolvePrNumber for a dependent task + const tid = (cmd.input.Key as { task_id: string }).task_id; + return { Item: { task_id: tid, pr_url: `https://github.com/o/r/pull/${tid.length}` } }; + } + if (cmd._type === 'BatchGet') { // resolveChildPrUrls for the panel + const keys = (cmd.input.RequestItems as Record }>); + const tbl = Object.keys(keys)[0]; + return { Responses: { [tbl]: keys[tbl].Keys.map((k) => ({ task_id: k.task_id, pr_url: `https://github.com/o/r/pull/${k.task_id.length}` })) } }; + } + return {}; + }); +} + +describe('feedback surface is chosen from the orchestration row, not assumed', () => { + // The reconciler is event-driven: it acts on an orchestration it LOADED, so the + // surface is a property of that row. Picking the wrong one would address a + // different tenant's API entirely. + beforeEach(() => { + ddbSend.mockReset(); + createTaskCoreMock.mockReset().mockResolvedValue({ statusCode: 201, body: '{}' }); + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + }); + + const completed = () => ({ + Records: [taskRecord({ task_id: 'TA', status: 'COMPLETED', orchestration_id: 'orch_1' })], + }) as never; + + test('a row recording the Linear channel drives the Linear surface', async () => { + mockOrchestration({ + subIssueId: 'A', + children: [{ sub_issue_id: 'A', child_status: 'released' }], + meta: { channel_source: 'linear' }, + }); + await handler(completed()); + expect(upsertStatusCommentMock).toHaveBeenCalled(); + }); + + test('a row with NO recorded channel still drives Linear (rows seeded before the field existed)', async () => { + mockOrchestration({ + subIssueId: 'A', + children: [{ sub_issue_id: 'A', child_status: 'released' }], + }); // no channel_source at all + await handler(completed()); + expect(upsertStatusCommentMock).toHaveBeenCalled(); + }); + + test('a row whose surface has no configured registry skips feedback instead of posting to Linear', async () => { + // The Jira tenant registry is unset in this handler's env, so a Jira-sourced + // orchestration has no adapter. It must stay silent — NOT fall through and + // address the Linear workspace, which is a different tenant's data. + mockOrchestration({ + subIssueId: 'A', + children: [{ sub_issue_id: 'A', child_status: 'released' }], + meta: { channel_source: 'jira' }, + }); + await handler(completed()); + expect(upsertStatusCommentMock).not.toHaveBeenCalled(); + expect(swapIssueReactionMock).not.toHaveBeenCalled(); + expect(transitionIssueStateMock).not.toHaveBeenCalled(); + }); +}); + +describe('orchestration-reconciler handler — cascading onto dependents', () => { + beforeEach(() => { + ddbSend.mockReset(); + createTaskCoreMock.mockReset(); + createTaskCoreMock.mockResolvedValue({ statusCode: 201, body: '{}' }); + postIssueCommentMock.mockReset().mockResolvedValue(true); + }); + + test('restack on B completes → re-stacks B\'s direct dependent C (one hop)', async () => { + // chain A→B→C, all started; the just-completed task re-stacked B. + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B' }, + { sub_issue_id: 'C', depends_on: ['B'], child_status: 'succeeded', child_task_id: 'task-C', child_branch_name: 'branch-C' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'restack-task-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })], + } as never); + + // Exactly one restack spawned — for C (B's direct dependent), NOT A. + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + const [body, ctx] = createTaskCoreMock.mock.calls[0]; + expect(body.workflow_ref).toBe('coding/restack-v1'); + expect(ctx.channelMetadata.orchestration_sub_issue_id).toBe('C'); + expect(ctx.channelMetadata.restack_predecessor_sub_issue_id).toBe('B'); + expect(ctx.channelMetadata.orchestration_merge_branches).toBe(JSON.stringify(['branch-B'])); + // Idempotency keyed on the SOURCE task id (converges, no loop). + expect(ctx.idempotencyKey).toContain('restack-task-1'); + }); + + test('iteration on A completes → re-stacks A\'s direct dependent B', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-task-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + })], + } as never); + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + expect(createTaskCoreMock.mock.calls[0][1].channelMetadata.orchestration_sub_issue_id).toBe('B'); + }); + + test('a cascade that RE-OPENS the epic clears rollup_posted_at (so parent state can re-settle)', async () => { + // A comment on an already-completed epic re-opens it. The first + // completion's rollup_posted_at stamp must be cleared, or claimRollup stays + // failed forever and the parent reaction/state never re-mirror (👀→✅). + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-task-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + })], + } as never); + // An Update issued a `REMOVE rollup_posted_at` on the meta row. + const clears = ddbSend.mock.calls + .map((c) => c[0]) + .filter((cmd) => cmd?._type === 'Update' + && typeof cmd.input?.UpdateExpression === 'string' + && cmd.input.UpdateExpression.includes('REMOVE rollup_posted_at')); + expect(clears.length).toBeGreaterThan(0); + }); + + test('FAILED iteration → no cascade', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-fail', + status: 'FAILED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + })], + } as never); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('cascade source with no started dependents → no restack', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'blocked' }, // not started + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + })], + } as never); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('a re-stack of a NO-DEPENDENTS node still refreshes the panel + settles (not stuck)', async () => { + // The hang this closes: a cascade source with no dependents returned + // early without refreshing → the node's '🔄 updating' row never cleared and + // the epic never re-settled to ✅. Here every child is already terminal, so + // the completion settle must fire: panel edited + parent state mirrored. + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + // B is a leaf (nothing depends on it) AND has no dependents → planDirectRestack=0. + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + // A re-stack of B (the no-dependents leaf) completes. + await handler({ + Records: [taskRecord({ + task_id: 'restack-B', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })], + } as never); + + // No further restack (B has no dependents). + expect(createTaskCoreMock).not.toHaveBeenCalled(); + // But the panel WAS refreshed (settle) — and since all children are + // terminal, it shows complete + mirrors parent state. + expect(upsertStatusCommentMock).toHaveBeenCalled(); + const body = upsertStatusCommentMock.mock.calls.at(-1)![2] as string; + expect(body).toMatch(/complete/i); + expect(body).not.toMatch(/updating/i); // the stale updating row is gone + expect(transitionIssueStateMock).toHaveBeenCalled(); // parent settled + }); + + test('a recorded retry comment is settled to ✅ when the re-run epic ends clean', async () => { + // The gap this closes: the retry path can only ack the comment (👀) — it + // returns the moment the work is dispatched — so nothing ever told the user + // whether their retry worked. Observed in practice: the comment sat on 👀 + // with no reply while the panel showed the finished epic right above it. + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ], { retry_comment_id: 'retry-cmt-1' }); + + await handler({ + Records: [taskRecord({ + task_id: 'restack-B', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })], + } as never); + + // The marker moves off 👀 — it is the whole answer, since the retry path posts + // no maturing reply. + expect(swapCommentReactionMock).toHaveBeenCalledWith(expect.anything(), 'retry-cmt-1', 'white_check_mark'); + // And the record is cleared, so a later settle can't re-swap an answered comment. + const cleared = ddbSend.mock.calls + .map((c) => c[0] as { _type?: string; input?: { UpdateExpression?: string } }) + .filter((cmd) => cmd?._type === 'Update' && /REMOVE retry_comment_id/.test(cmd.input?.UpdateExpression ?? '')); + expect(cleared).toHaveLength(1); + }); + + test('a retry that ends with failures settles the comment to ❌, matching the panel header', async () => { + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'failed', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ], { retry_comment_id: 'retry-cmt-1' }); + + await handler({ + Records: [taskRecord({ + task_id: 'restack-B', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })], + } as never); + + expect(swapCommentReactionMock).toHaveBeenCalledWith(expect.anything(), 'retry-cmt-1', 'x'); + }); + + test('an epic still in flight does NOT settle the retry comment yet', async () => { + // Premature settling would tell the user "done" while children are still + // running. The cascade source here is a LEAF (nothing depends on it), which is + // the shape that routes through the settle path — a source WITH dependents + // refreshes the panel from the cascade branch instead and never reaches it, so + // that fixture would pass this assertion without exercising anything. + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + // B: the leaf whose restack just completed. C is still running, so the epic + // as a whole is NOT terminal. + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + { sub_issue_id: 'C', child_status: 'released', child_task_id: 'task-C', child_branch_name: 'branch-C', linear_identifier: 'ENG-3' }, + ], { retry_comment_id: 'retry-cmt-1' }); + + await handler({ + Records: [taskRecord({ + task_id: 'restack-B', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })], + } as never); + + // The settle path DID run — the panel shows the in-flight header (🔄 · n/m), + // not a settled one — so it reached the gate and declined, rather than this + // test never reaching the code. + expect(upsertStatusCommentMock).toHaveBeenCalled(); + expect(upsertStatusCommentMock.mock.calls.at(-1)![2] as string).toMatch(/^🔄/); + expect(swapCommentReactionMock).not.toHaveBeenCalledWith(expect.anything(), 'retry-cmt-1', expect.anything()); + }); + + test('an epic with NO recorded retry comment settles nothing extra', async () => { + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + + await handler({ + Records: [taskRecord({ + task_id: 'restack-B', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'B', + restack_predecessor_sub_issue_id: 'A', + })], + } as never); + + const cleared = ddbSend.mock.calls + .map((c) => c[0] as { _type?: string; input?: { UpdateExpression?: string } }) + .filter((cmd) => cmd?._type === 'Update' && /retry_comment_id/.test(cmd.input?.UpdateExpression ?? '')); + expect(cleared).toHaveLength(0); + }); + + test('a cascade source does NOT run normal child gating (no GSI sub-issue lookup)', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + })], + } as never); + // Never queried ChildTaskIndex (that's the normal-gating path). + const gsiCalls = ddbSend.mock.calls.filter( + (c) => c[0]?._type === 'Query' && c[0]?.input?.IndexName === 'ChildTaskIndex'); + expect(gsiCalls).toHaveLength(0); + }); +}); + +describe('orchestration-reconciler handler — cascade surfacing via the panel', () => { + beforeEach(() => { + ddbSend.mockReset(); + createTaskCoreMock.mockReset().mockResolvedValue({ statusCode: 201, body: '{}' }); + postIssueCommentMock.mockReset().mockResolvedValue(true); + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + }); + + const iterEvent = (sub: string) => ({ + Records: [taskRecord({ + task_id: 'iter-task-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: sub, + orchestration_iteration: true, + })], + }) as never; + + test('refreshes the panel with the impacted row as "updating per comment" — NO standalone parent/sub-issue comments', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + await handler(iterEvent('A')); + // The panel is edited (upsertStatusComment), NOT a stream of new comments. + expect(upsertStatusCommentMock).toHaveBeenCalled(); + const body = upsertStatusCommentMock.mock.calls.at(-1)![2] as string; + // Impacted dependent B shows '🔄 … updating per ENG-1's comment'. + expect(body).toMatch(/ENG-2.*updating per ENG-1's comment/); + // The retired standalone '🔄 Re-stacked' / 'revised' parent comments are GONE. + expect(postIssueCommentMock).not.toHaveBeenCalled(); + }); + + test('idempotent replay (200, NOT 201) does NOT re-mark the panel as updating', async () => { + createTaskCoreMock.mockResolvedValue({ statusCode: 200, body: '{}' }); + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + await handler(iterEvent('A')); + // No NEW restack task created → no panel "updating" refresh from the cascade. + expect(upsertStatusCommentMock).not.toHaveBeenCalled(); + }); + + test('integration-node dependent renders friendly in the panel (never raw id)', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'orch_1__integration', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-int', child_branch_name: 'branch-int' }, + ]); + await handler(iterEvent('A')); + expect(upsertStatusCommentMock).toHaveBeenCalled(); + const body = upsertStatusCommentMock.mock.calls.at(-1)![2] as string; + expect(body).toContain('Integration — combined result'); + expect(body).not.toContain('orch_1__integration'); + }); + + test('a restack from a PREDECESSOR change (not a comment) says "updating to include … change"', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + // restack source (carries restack_predecessor, NOT orchestration_iteration). + await handler({ + Records: [taskRecord({ + task_id: 'restack-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + restack_predecessor_sub_issue_id: 'Z', + })], + } as never); + const body = upsertStatusCommentMock.mock.calls.at(-1)![2] as string; + expect(body).toMatch(/ENG-2.*updating to include ENG-1's change/); + }); +}); + +describe('orchestration-reconciler handler — the iteration ack reply', () => { + beforeEach(() => { + ddbSend.mockReset(); + createTaskCoreMock.mockReset().mockResolvedValue({ statusCode: 201, body: '{}' }); + postIssueCommentMock.mockReset().mockResolvedValue(true); + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-cmt-1'); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + swapCommentReactionMock.mockReset().mockResolvedValue(true); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + replyToCommentMock.mockReset().mockResolvedValue('reply-1'); + upsertThreadedReplyMock.mockReset().mockResolvedValue('reply-1'); + }); + + /** An iteration event carrying the human comment id that triggered it. */ + const iterEventWithComment = (status: string, commentId = 'human-cmt-1', buildPassed?: boolean, errorMessage?: string) => ({ + Records: [taskRecord({ + task_id: 'iter-task-1', + status, + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + trigger_comment_id: commentId, + ...(buildPassed !== undefined && { build_passed: buildPassed }), + ...(errorMessage !== undefined && { error_message: errorMessage }), + })], + }) as never; + + test('successful iteration → ✅ threaded reply to the triggering comment, linking the PR', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler(iterEventWithComment('COMPLETED')); + + expect(upsertThreadedReplyMock).toHaveBeenCalledTimes(1); + // Signature: replyToComment(ctx, issueId, parentCommentId, body). + const [, issueId, parentCommentId, body] = upsertThreadedReplyMock.mock.calls[0]; + expect(issueId).toBe('A'); // the sub-issue the comment lives on + expect(parentCommentId).toBe('human-cmt-1'); + // The PR ref is a clickable markdown link when the URL resolves. + expect(body).toMatch(/^✅ Updated — \[PR #\d+\]\(https:\/\/.*\)\./); + // The trigger comment's 👀 swaps to ✅, and the sub-issue + // advances to In Review (platform-owned settle, not agent-flapped). + expect(swapCommentReactionMock).toHaveBeenCalledWith(expect.anything(), 'human-cmt-1', 'white_check_mark'); + // The channel passes the same-category-regression flag explicitly; a plain + // advance never allows one. + expect(transitionIssueStateMock).toHaveBeenCalledWith(expect.anything(), 'A', 'started', ['In Review'], false); + }); + + test('a no-change iteration (a question) settles 💬 and leaves the sub-issue state alone', async () => { + // An answer is neither a success-edit nor a failure. ✅ would imply "PR + // updated, merge-worthy" and advancing the sub-issue would claim work landed + // — nothing changed, so the comment gets ❓ and the state is untouched. + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-task-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + trigger_comment_id: 'human-cmt-1', + code_changed: false, // the agent answered without editing anything + })], + } as never); + + expect(swapCommentReactionMock).toHaveBeenCalledWith(expect.anything(), 'human-cmt-1', 'question'); + // The SUB-ISSUE is not advanced. (The parent epic still settles on its own, + // since every child is terminal — that's the panel mirror, not this answer.) + expect(transitionIssueStateMock).not.toHaveBeenCalledWith( + expect.anything(), 'A', expect.anything(), expect.anything(), expect.anything(), + ); + }); + + test('a PARENT-routed iteration replies on the PARENT issue, not the sub-issue', async () => { + // The human commented on the parent epic and the router resolved it to + // sub-issue A. The ✅/❌ reply must use the PARENT issue id as + // commentCreate's issueId — else Linear rejects the reply (parentId belongs + // to a different issue) and the human sees 👀 then silence. + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'iter-task-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + orchestration_iteration: true, + trigger_comment_id: 'parent-cmt-1', + trigger_comment_issue_id: 'PARENT', // comment lives on the parent epic + })], + } as never); + + expect(upsertThreadedReplyMock).toHaveBeenCalledTimes(1); + const [, issueId, parentCommentId] = upsertThreadedReplyMock.mock.calls[0]; + expect(issueId).toBe('PARENT'); // NOT 'A' — the reply targets the parent comment's issue + expect(parentCommentId).toBe('parent-cmt-1'); + }); + + test('a FAILED iteration (agent crash) → ❌ reply with a classified reason + the CloudWatch task id', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler(iterEventWithComment('FAILED', 'human-cmt-1', undefined, 'agent_status="error_max_turns"')); + + expect(upsertThreadedReplyMock).toHaveBeenCalledTimes(1); + const [, , , body] = upsertThreadedReplyMock.mock.calls[0]; + expect(body).toMatch(/^❌/); + expect(body).toMatch(/Exceeded max turns/i); // classified + expect(body).toMatch(/CloudWatch for task `iter-task-1`/); + // retryable agent/timeout → plain reply-to-retry next step (retryGuidance). + expect(body).toMatch(/reply here with any extra guidance/i); + // A failed iteration still does not cascade onto dependents. + expect(createTaskCoreMock).not.toHaveBeenCalled(); + // The trigger comment's 👀 swaps to ❌, but the sub-issue state + // is LEFT in place on failure (the ❌ + reply convey it; never demote). + expect(swapCommentReactionMock).toHaveBeenCalledWith(expect.anything(), 'human-cmt-1', 'x'); + expect(transitionIssueStateMock).not.toHaveBeenCalled(); + }); + + test('a COMPLETED-but-build-failed iteration → ❌ build/test reply pointing at the build log in CloudWatch', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + // COMPLETED, build_passed=false, NO error_message → build/test failure shape. + await handler(iterEventWithComment('COMPLETED', 'human-cmt-1', false)); + + expect(upsertThreadedReplyMock).toHaveBeenCalledTimes(1); + const [, , , body] = upsertThreadedReplyMock.mock.calls[0]; + expect(body).toMatch(/build\/tests didn't pass/i); + // Build-gate failures point at the agent's CloudWatch build log + // (the build ran in the microVM), not the PR's GitHub checks. + expect(body).toMatch(/build log in CloudWatch/i); + expect(body).not.toMatch(/PR's checks/i); + // build_passed=false ⇒ not a success ⇒ no cascade onto dependents. + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('build_passed=false → ❌ reply (treated as not-successful)', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler(iterEventWithComment('COMPLETED', 'human-cmt-1', false)); + const [, , , body] = upsertThreadedReplyMock.mock.calls[0]; + expect(body).toMatch(/^❌/); + }); + + test('idempotent: redelivery loses the claim → no duplicate reply', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + // First Update (the ack claim) wins; a second Update with the same key is + // rejected by the conditional → simulate the redelivery losing the claim. + let ackClaims = 0; + const base = ddbSend.getMockImplementation()!; + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + if (cmd._type === 'Update' && (cmd.input.UpdateExpression as string)?.includes('ack_replied_at')) { + ackClaims += 1; + if (ackClaims > 1) { + const err = new Error('conditional'); + (err as { name?: string }).name = 'ConditionalCheckFailedException'; + throw err; + } + return {}; + } + return base(cmd); + }); + + await handler(iterEventWithComment('COMPLETED')); + await handler(iterEventWithComment('COMPLETED')); // redelivery + + // Replied exactly once across both deliveries. + expect(upsertThreadedReplyMock).toHaveBeenCalledTimes(1); + }); + + test('the terminal settle asks the surface to restore the outcome if it gets overwritten', async () => { + // The progress writers avoid clobbering by reading the body first, but that is + // a read then a separate write, and the surface offers no conditional update + // to close the gap — so the writer holding the outcome verifies afterwards. + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler(iterEventWithComment('COMPLETED')); + + const options = upsertThreadedReplyMock.mock.calls[0][5] as Record; + expect(options).toMatchObject({ preservePreview: true, repairIfOverwritten: true }); + // A terminal render must never yield to progress; only the reverse. + expect(options.skipIfSettled).toBeFalsy(); + }); + + test('a FAILED reply hands the claim back and defers the settle while attempts remain', async () => { + // Two hazards if the claim is kept: no redelivery can retry the reply, and the + // progress + heartbeat writers treat the claim as "an outcome landed" and stand + // down — so the reply stalls on its last progress text. Settling the comment to + // ✅ on top of that would be worse still: the reaction would say done while the + // reply says working. + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + upsertThreadedReplyMock.mockReset().mockResolvedValue(null); + + await handler(iterEventWithComment('COMPLETED')); + + const acks = ddbSend.mock.calls + .map((c) => c[0] as { _type?: string; input?: { UpdateExpression?: string; ExpressionAttributeValues?: Record } }) + .filter((cmd) => cmd?._type === 'Update' && /ack_replied_at/.test(cmd.input?.UpdateExpression ?? '')); + expect(acks[0].input?.UpdateExpression).toBe('SET ack_replied_at = :now'); + expect(acks[1].input?.UpdateExpression).toContain('REMOVE ack_replied_at'); + // Released conditionally on this run's own stamp, so a concurrent delivery that + // has already claimed and replied keeps its claim. + expect(acks[1].input?.ExpressionAttributeValues?.[':ours']) + .toBe(acks[0].input?.ExpressionAttributeValues?.[':now']); + expect(swapCommentReactionMock).not.toHaveBeenCalled(); + expect(transitionIssueStateMock).not.toHaveBeenCalledWith( + expect.anything(), 'A', 'started', ['In Review'], false, + ); + }); + + test('once the retry budget is spent it settles the REACTION anyway, rather than leaving 👀', async () => { + // Observed in practice: with a reply that could never succeed, the release + // re-woke this handler (it writes to the task record, whose stream feeds it) + // ~900 times, and the trigger comment sat on 👀 with no reply at all. A + // permanent "still working" is worse than an outcome carried by the marker + // alone. + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + upsertThreadedReplyMock.mockReset().mockResolvedValue(null); + const base = ddbSend.getMockImplementation()!; + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + const expr = String(cmd.input?.UpdateExpression ?? ''); + if (cmd._type === 'Update' && expr.includes('REMOVE ack_replied_at')) { + const err = new Error('conditional'); + (err as { name?: string }).name = 'ConditionalCheckFailedException'; + throw err; + } + // The budget re-read that distinguishes "spent" from "not ours". + if (cmd._type === 'Get' && cmd.input?.ProjectionExpression === 'ack_reply_attempts') { + return { Item: { ack_reply_attempts: 3 } }; + } + return base(cmd); + }); + + await handler(iterEventWithComment('COMPLETED')); + + // The outcome still reaches the human, as a reaction. + expect(swapCommentReactionMock).toHaveBeenCalledWith(expect.anything(), 'human-cmt-1', 'white_check_mark'); + }); + + test('a SUCCESSFUL reply keeps the claim, so the once-only guarantee survives the fix', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + ]); + await handler(iterEventWithComment('COMPLETED')); + + const released = ddbSend.mock.calls + .map((c) => c[0] as { _type?: string; input?: { UpdateExpression?: string } }) + .filter((cmd) => cmd?._type === 'Update' && /REMOVE ack_replied_at/.test(cmd.input?.UpdateExpression ?? '')); + expect(released).toHaveLength(0); + expect(swapCommentReactionMock).toHaveBeenCalled(); + }); + + test('a restack (no trigger_comment_id) → no ack reply', async () => { + mockCascade([ + { sub_issue_id: 'A', child_status: 'succeeded', child_task_id: 'task-A', child_branch_name: 'branch-A', linear_identifier: 'ENG-1' }, + { sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', child_branch_name: 'branch-B', linear_identifier: 'ENG-2' }, + ]); + await handler({ + Records: [taskRecord({ + task_id: 'restack-1', + status: 'COMPLETED', + orchestration_id: 'orch_1', + orchestration_sub_issue_id: 'A', + restack_predecessor_sub_issue_id: 'Z', + })], + } as never); + expect(upsertThreadedReplyMock).not.toHaveBeenCalled(); + }); +}); + +describe('the panel preview on a CHAIN (no integration node)', () => { + test('reads the preview off the final node instead of showing none', async () => { + // A chain has ONE leaf, so no integration node is seeded — by then everything + // has converged on the last child, and a synthetic node would re-run what that + // child just did. But the reviewer reads the PARENT panel, so "no integration + // node" must not mean "no preview": the final node holds exactly the artifact + // they want. Previously the panel showed nothing at all on a chain. + upsertStatusCommentMock.mockReset().mockResolvedValue('panel-1'); + transitionIssueStateMock.mockReset().mockResolvedValue(true); + swapIssueReactionMock.mockReset().mockResolvedValue(true); + const meta = { + sub_issue_id: '#meta', + orchestration_id: 'orch_1', + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + child_count: 2, + platform_user_id: 'u1', + status_comment_id: 'panel-1', + }; + const base = { orchestration_id: 'orch_1', repo: 'o/r', parent_linear_issue_id: 'PARENT', linear_workspace_id: 'WS' }; + // A → B: B is the sole leaf and the one that just completed. + const rows = [ + { ...base, sub_issue_id: 'A', depends_on: [], child_status: 'succeeded', child_task_id: 'task-A', linear_identifier: 'ENG-1' }, + { ...base, sub_issue_id: 'B', depends_on: ['A'], child_status: 'succeeded', child_task_id: 'task-B', linear_identifier: 'ENG-2' }, + ]; + ddbSend.mockImplementation(async (cmd: { _type: string; input: Record }) => { + if (cmd._type === 'Query' && cmd.input.IndexName === 'ChildTaskIndex') return { Items: [{ ...rows[1] }] }; + if (cmd._type === 'Query') return { Items: [meta, ...rows] }; + if (cmd._type === 'BatchGet') { + const keys = cmd.input.RequestItems as Record }>; + const tbl = Object.keys(keys)[0]; + return { Responses: { [tbl]: keys[tbl].Keys.map((k) => ({ task_id: k.task_id, pr_url: `https://github.com/o/r/pull/${k.task_id.length}` })) } }; + } + if (cmd._type === 'Get') { + const tid = (cmd.input.Key as { task_id: string }).task_id; + // Only the FINAL node carries a screenshot; the first child's is stale. + return { + Item: tid === 'task-B' + ? { screenshot_url: 'https://cdn.example/final.png', screenshot_preview_url: 'https://final.vercel.app' } + : { screenshot_url: 'https://cdn.example/WRONG-first-child.png' }, + }; + } + return {}; + }); + + await handler({ + Records: [{ + eventName: 'MODIFY', + dynamodb: { + NewImage: { + task_id: { S: 'task-B' }, + status: { S: 'COMPLETED' }, + orchestration_id: { S: 'orch_1' }, + build_passed: { BOOL: true }, + } as never, + }, + }], + } as never); + + const panel = upsertStatusCommentMock.mock.calls.map((c) => String(c[2])).join('\n'); + expect(panel).toContain('https://cdn.example/final.png'); + expect(panel).not.toContain('WRONG-first-child'); + // Clickable deep-link to the running site, not just a static PNG. + expect(panel).toContain('https://final.vercel.app'); + // NOT presented as a combined result — nothing was merged on a chain. + expect(panel).toContain('🖼️ **Preview**'); + expect(panel).not.toContain('Combined preview'); + }); +}); diff --git a/cdk/test/handlers/reconcile-stranded-orchestrations.test.ts b/cdk/test/handlers/reconcile-stranded-orchestrations.test.ts new file mode 100644 index 000000000..6bbad621c --- /dev/null +++ b/cdk/test/handlers/reconcile-stranded-orchestrations.test.ts @@ -0,0 +1,249 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Stranded-orchestration backstop. Uses a stateful in-memory + * DynamoDB fake so the sweep's read-advance-release cycle is exercised + * for real (status writes are visible to the subsequent reload). + */ + +interface Row { [k: string]: unknown } +const orch = new Map(); // OrchestrationTable, key = `${oid} ${sk}` +const tasksTbl = new Map(); // TaskTable, key = task_id + +const fakeSend = jest.fn(async (cmd: { _type: string; input: Record }) => { + const { _type, input } = cmd; + const tn = input.TableName as string; + if (_type === 'Scan') { + // meta-row scan on OrchestrationTable + const items = [...orch.values()].filter((r) => r.sub_issue_id === '#meta'); + return { Items: items }; + } + if (_type === 'Get') { + const k = input.Key as Row; + return { Item: tn.includes('Task') ? tasksTbl.get(String(k.task_id)) : orch.get(`${k.orchestration_id} ${k.sub_issue_id}`) }; + } + if (_type === 'Query') { + const oid = (input.ExpressionAttributeValues as Row)[':oid']; + return { Items: [...orch.values()].filter((r) => r.orchestration_id === oid) }; + } + if (_type === 'Update') { + const k = input.Key as Row; + const key = `${k.orchestration_id} ${k.sub_issue_id}`; + const vals = input.ExpressionAttributeValues as Row; + const row = orch.get(key); + if (row && input.ConditionExpression?.toString().includes('child_status <> :s') && row.child_status === vals[':s']) { + const e = new Error('c'); e.name = 'ConditionalCheckFailedException'; throw e; + } + if (row) { + if (vals[':s'] !== undefined) row.child_status = vals[':s']; + if (vals[':released'] !== undefined) { row.child_status = 'released'; row.child_task_id = vals[':tid']; } + } + return {}; + } + throw new Error(`fake: unhandled ${_type}`); +}); + +jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: fakeSend })) }, + ScanCommand: jest.fn((input: unknown) => ({ _type: 'Scan', input })), + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), + QueryCommand: jest.fn((input: unknown) => ({ _type: 'Query', input })), + UpdateCommand: jest.fn((input: unknown) => ({ _type: 'Update', input })), +})); + +const createTaskCoreMock = jest.fn(); +jest.mock('../../src/handlers/shared/create-task-core', () => ({ + createTaskCore: (...args: unknown[]) => createTaskCoreMock(...args), +})); +jest.mock('../../src/handlers/shared/logger', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +// The sweep imports refreshPanelAndSettle from the reconciler to settle an epic +// whose last terminal event was lost. Mock it so we can assert the sweep CALLS +// it (its real body is exercised by the reconciler's own suite). +const refreshPanelAndSettleMock = jest.fn(); +jest.mock('../../src/handlers/orchestration-reconciler', () => ({ + refreshPanelAndSettle: (...args: unknown[]) => refreshPanelAndSettleMock(...args), +})); + +process.env.ORCHESTRATION_TABLE_NAME = 'OrchestrationTable'; +process.env.TASK_TABLE_NAME = 'TaskTable'; + +import { handler } from '../../src/handlers/reconcile-stranded-orchestrations'; + +function seed(oid: string, children: Array<{ sk: string; deps?: string[]; status: string; taskId?: string }>): void { + orch.set(`${oid} #meta`, { + orchestration_id: oid, + sub_issue_id: '#meta', + parent_linear_issue_id: 'P', + linear_workspace_id: 'WS', + repo: 'o/r', + child_count: children.length, + platform_user_id: 'user-1', + }); + for (const c of children) { + orch.set(`${oid} ${c.sk}`, { + orchestration_id: oid, + sub_issue_id: c.sk, + depends_on: c.deps ?? [], + child_status: c.status, + repo: 'o/r', + parent_linear_issue_id: 'P', + linear_workspace_id: 'WS', + ...(c.taskId && { child_task_id: c.taskId }), + }); + } +} +const statusOf = (oid: string, sk: string) => orch.get(`${oid} ${sk}`)?.child_status; + +beforeEach(() => { + orch.clear(); tasksTbl.clear(); fakeSend.mockClear(); + createTaskCoreMock.mockReset(); + createTaskCoreMock.mockResolvedValue({ statusCode: 201, body: JSON.stringify({ data: { task_id: 'new-task' } }) }); + refreshPanelAndSettleMock.mockReset(); +}); + +describe('stranded-orchestration backstop', () => { + test('lost RELEASE event: A already succeeded, B blocked → sweep releases B', async () => { + // The reconciler missed releasing B even though A is succeeded. + seed('o1', [ + { sk: 'A', status: 'succeeded' }, + { sk: 'B', deps: ['A'], status: 'blocked' }, + ]); + await handler(); + expect(createTaskCoreMock).toHaveBeenCalledTimes(1); + expect(statusOf('o1', 'B')).toBe('released'); + }); + + test('lost TERMINAL event: A released + its task COMPLETED but row stuck → sweep advances A and releases B', async () => { + seed('o2', [ + { sk: 'A', status: 'released', taskId: 'task-A' }, + { sk: 'B', deps: ['A'], status: 'blocked' }, + ]); + tasksTbl.set('task-A', { task_id: 'task-A', status: 'COMPLETED', build_passed: true }); + await handler(); + expect(statusOf('o2', 'A')).toBe('succeeded'); + expect(statusOf('o2', 'B')).toBe('released'); + }); + + // The sweep must SETTLE the parent (post rollup + transition Linear), not + // just flip DDB rows — else a lost last-terminal event leaves the epic 'done' + // in DDB but stuck 🔄 In Progress in Linear forever. + test('lost LAST terminal event: the only remaining child completes → sweep settles the parent', async () => { + seed('o-settle', [ + { sk: 'A', status: 'succeeded' }, + { sk: 'B', deps: ['A'], status: 'released', taskId: 'task-B' }, + ]); + tasksTbl.set('task-B', { task_id: 'task-B', status: 'COMPLETED', build_passed: true }); + await handler(); + // B advanced to terminal in DDB… + expect(statusOf('o-settle', 'B')).toBe('succeeded'); + // …and, crucially, the parent was settled — the whole point of the backstop. + expect(refreshPanelAndSettleMock).toHaveBeenCalled(); + expect(createTaskCoreMock).not.toHaveBeenCalled(); // nothing left to release + }); + + test('already fully-terminal but unsettled epic → sweep still settles (idempotent)', async () => { + seed('o-done', [ + { sk: 'A', status: 'succeeded' }, + { sk: 'B', deps: ['A'], status: 'succeeded' }, + ]); + await handler(); + expect(refreshPanelAndSettleMock).toHaveBeenCalledWith('o-done', expect.any(Array), expect.any(Object), expect.any(String)); + }); + + test('lost TERMINAL event with build_passed=false: A→failed, B→skipped, no release', async () => { + seed('o3', [ + { sk: 'A', status: 'released', taskId: 'task-A' }, + { sk: 'B', deps: ['A'], status: 'blocked' }, + ]); + tasksTbl.set('task-A', { task_id: 'task-A', status: 'COMPLETED', build_passed: false }); + await handler(); + expect(statusOf('o3', 'A')).toBe('failed'); + expect(statusOf('o3', 'B')).toBe('skipped'); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('transitive skip: A failed → B and C (chain) both skipped', async () => { + seed('o4', [ + { sk: 'A', status: 'failed' }, + { sk: 'B', deps: ['A'], status: 'blocked' }, + { sk: 'C', deps: ['B'], status: 'blocked' }, + ]); + await handler(); + expect(statusOf('o4', 'B')).toBe('skipped'); + expect(statusOf('o4', 'C')).toBe('skipped'); + }); + + test('still-running child is left alone (task not terminal)', async () => { + seed('o5', [ + { sk: 'A', status: 'released', taskId: 'task-A' }, + { sk: 'B', deps: ['A'], status: 'blocked' }, + ]); + tasksTbl.set('task-A', { task_id: 'task-A', status: 'RUNNING' }); + await handler(); + expect(statusOf('o5', 'A')).toBe('released'); // unchanged + expect(statusOf('o5', 'B')).toBe('blocked'); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('fully-terminal orchestration is skipped (no work, no release)', async () => { + seed('o6', [ + { sk: 'A', status: 'succeeded' }, + { sk: 'B', deps: ['A'], status: 'succeeded' }, + ]); + await handler(); + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); + + test('diamond: D releases only once BOTH B and C are succeeded', async () => { + seed('o7', [ + { sk: 'B', status: 'succeeded' }, + { sk: 'C', status: 'succeeded' }, + { sk: 'D', deps: ['B', 'C'], status: 'blocked' }, + ]); + await handler(); + expect(statusOf('o7', 'D')).toBe('released'); + }); + + test('diamond not-ready: one predecessor still running → D stays blocked', async () => { + seed('o8', [ + { sk: 'B', status: 'succeeded' }, + { sk: 'C', status: 'released', taskId: 'task-C' }, + { sk: 'D', deps: ['B', 'C'], status: 'blocked' }, + ]); + tasksTbl.set('task-C', { task_id: 'task-C', status: 'RUNNING' }); + await handler(); + expect(statusOf('o8', 'D')).toBe('blocked'); + }); + + test('idempotent: a second sweep over a healthy orchestration releases nothing new', async () => { + seed('o9', [ + { sk: 'A', status: 'succeeded' }, + { sk: 'B', deps: ['A'], status: 'blocked' }, + ]); + await handler(); + createTaskCoreMock.mockClear(); + await handler(); // B is now 'released' → no further release + expect(createTaskCoreMock).not.toHaveBeenCalled(); + }); +}); diff --git a/cdk/test/handlers/shared/iteration-cost.test.ts b/cdk/test/handlers/shared/iteration-cost.test.ts new file mode 100644 index 000000000..7b502f7d9 --- /dev/null +++ b/cdk/test/handlers/shared/iteration-cost.test.ts @@ -0,0 +1,156 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +const send = jest.fn(); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + QueryCommand: jest.fn((input: unknown) => ({ _type: 'Query', input })), + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), +})); + +import type { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { sumIterationCostForIssue } from '../../../src/handlers/shared/iteration-cost'; + +const ddb = { send } as unknown as DynamoDBDocumentClient; + +const call = (over: Partial[0]> = {}) => + sumIterationCostForIssue({ + ddb, taskTableName: 'Tasks', linearIssueId: 'issue-1', thisTaskId: 'task-this', ...over, + }); + +/** A Query page listing task ids, optionally with a continuation key. */ +const page = (ids: string[], last?: Record) => ({ + Items: ids.map((task_id) => ({ task_id })), + ...(last && { LastEvaluatedKey: last }), +}); + +const isQuery = (c: { _type?: string }) => c._type === 'Query'; + +beforeEach(() => { + send.mockReset(); +}); + +describe('sumIterationCostForIssue', () => { + test('sums prior tasks plus this task, and reports a COMPLETE total', async () => { + send + .mockResolvedValueOnce(page(['task-a', 'task-this'])) + .mockResolvedValueOnce({ Item: { cost_usd: 1.5 } }); // task-a + + await expect(call({ thisCost: 0.25 })).resolves.toEqual({ total: 1.75, partial: false }); + // this task's cost comes from the argument, so no GetItem is spent on it + expect(send.mock.calls.filter(([c]) => !isQuery(c))).toHaveLength(1); + }); + + test('FOLLOWS the Query pagination cursor — a total must not stop at one 1 MB page', async () => { + // The pre-refactor version summed a single page as though it were everything, + // so past the page boundary the user saw a silently short total. + send + .mockResolvedValueOnce(page(['task-a'], { task_id: 'task-a' })) + .mockResolvedValueOnce(page(['task-b'])) + .mockResolvedValueOnce({ Item: { cost_usd: 2 } }) + .mockResolvedValueOnce({ Item: { cost_usd: 3 } }); + + await expect(call()).resolves.toEqual({ total: 5, partial: false }); + expect(send.mock.calls.filter(([c]) => isQuery(c))).toHaveLength(2); + }); + + test('a string cost_usd is parsed, not concatenated or turned into NaN', async () => { + // One of the two pre-refactor copies added a stringified cost without a finite + // check, which poisoned the whole total to NaN. + send + .mockResolvedValueOnce(page(['task-a'])) + .mockResolvedValueOnce({ Item: { cost_usd: '2.5' } }); + + await expect(call()).resolves.toEqual({ total: 2.5, partial: false }); + }); + + test('an unparseable cost is skipped rather than poisoning the total', async () => { + send + .mockResolvedValueOnce(page(['task-a', 'task-b'])) + .mockResolvedValueOnce({ Item: { cost_usd: 'not-a-number' } }) + .mockResolvedValueOnce({ Item: { cost_usd: 4 } }); + + await expect(call()).resolves.toEqual({ total: 4, partial: false }); + }); + + test('the row cap BOUNDS the GetItem fan-out and reports PARTIAL, even within one page', async () => { + // A single Query page can hold far more than the cap: the projection is + // task_id alone, so ~1 MB is a great many rows. Stopping pagination is not + // enough — the cap must also trim the id list the reads iterate, or a + // pathological issue still fans out over the whole page inside one invocation. + send + .mockResolvedValueOnce(page(Array.from({ length: 900 }, (_, i) => `t${i}`))) + .mockResolvedValue({ Item: { cost_usd: 1 } }); + + const res = await call(); + expect(res.partial).toBe(true); + expect(res.total).toBe(500); + expect(send.mock.calls.filter(([c]) => !isQuery(c))).toHaveLength(500); + }); + + test('the cap applies across pages too', async () => { + send + .mockResolvedValueOnce(page(Array.from({ length: 400 }, (_, i) => `t${i}`), { task_id: 't399' })) + .mockResolvedValueOnce(page(Array.from({ length: 400 }, (_, i) => `u${i}`))) + .mockResolvedValue({ Item: { cost_usd: 1 } }); + + const res = await call(); + expect(res.partial).toBe(true); + expect(res.total).toBe(500); + // stops after the page that crosses the cap — a third Query is never issued + expect(send.mock.calls.filter(([c]) => isQuery(c))).toHaveLength(2); + }); + + test('a failed read reports PARTIAL and still returns what it has', async () => { + send + .mockResolvedValueOnce(page(['task-a', 'task-b'])) + .mockResolvedValueOnce({ Item: { cost_usd: 1 } }) + .mockRejectedValueOnce(new Error('throttled')); + + await expect(call({ thisCost: 0.5 })).resolves.toEqual({ total: 1.5, partial: true }); + }); + + test('a failed Query reports PARTIAL and falls back to this task alone', async () => { + send.mockRejectedValueOnce(new Error('index unavailable')); + await expect(call({ thisCost: 0.75 })).resolves.toEqual({ total: 0.75, partial: true }); + }); + + test('nothing known → null total, not a misleading $0', async () => { + send.mockResolvedValueOnce(page([])); + await expect(call()).resolves.toEqual({ total: null, partial: false }); + }); + + test('this task is counted once even when the GSI already lists it', async () => { + send.mockResolvedValueOnce(page(['task-this'])); + await expect(call({ thisCost: 3 })).resolves.toEqual({ total: 3, partial: false }); + expect(send.mock.calls.filter(([c]) => !isQuery(c))).toHaveLength(0); + }); + + test('queries the LinearIssueIndex for the issue, projecting only task_id', async () => { + send.mockResolvedValueOnce(page([])); + await call(); + const [[cmd]] = send.mock.calls; + expect(cmd.input).toMatchObject({ + TableName: 'Tasks', + IndexName: 'LinearIssueIndex', + KeyConditionExpression: 'linear_issue_id = :iid', + ProjectionExpression: 'task_id', + ExpressionAttributeValues: { ':iid': 'issue-1' }, + }); + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-discovery.test.ts b/cdk/test/handlers/shared/orchestration-discovery.test.ts new file mode 100644 index 000000000..897d0472e --- /dev/null +++ b/cdk/test/handlers/shared/orchestration-discovery.test.ts @@ -0,0 +1,304 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { discoverOrchestration } from '../../../src/handlers/shared/orchestration-discovery'; +import { declarativeGraphSource, linearGraphSource } from '../../../src/handlers/shared/orchestration-graph-source'; + +jest.mock('../../../src/handlers/shared/logger', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +/** Mock fetch returning a Linear children payload. */ +function mockFetch(children: Array<{ id: string; blockedBy?: string[] }>): typeof fetch { + return (async () => ({ + ok: true, + status: 200, + json: async () => ({ + data: { + issue: { + id: 'PARENT', + children: { + nodes: children.map((c) => ({ + id: c.id, + inverseRelations: { nodes: (c.blockedBy ?? []).map((b) => ({ type: 'blocks', issue: { id: b } })) }, + })), + }, + }, + }, + }), + })) as unknown as typeof fetch; +} + +function errorFetch(): typeof fetch { + return (async () => ({ ok: false, status: 500, json: async () => ({}) })) as unknown as typeof fetch; +} + +function emptyFetch(): typeof fetch { + return (async () => ({ + ok: true, + status: 200, + json: async () => ({ data: { issue: { id: 'PARENT', children: { nodes: [] } } } }), + })) as unknown as typeof fetch; +} + +const base = { + tableName: 'OrchestrationTable', + accessToken: 'tok', + parentIssueRef: 'PARENT', + credentialsRef: 'WS', + repo: 'o/r', + now: '2026-06-09T12:00:00.000Z', + releaseContext: { platform_user_id: 'platform-user-1' }, +}; + +describe('discoverOrchestration', () => { + test('no sub-issues → single_task', async () => { + const ddb = { send: jest.fn() }; + const result = await discoverOrchestration({ + ...base, ddb: ddb as never, graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: emptyFetch() }), + }); + expect(result.kind).toBe('single_task'); + expect(ddb.send).not.toHaveBeenCalled(); // never touches the table + }); + + test('valid DAG → seeded with roots from layer 0', async () => { + const ddb = { send: jest.fn().mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}) }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }, { id: 'B', blockedBy: ['A'] }]) }), + }); + expect(result.kind).toBe('seeded'); + if (result.kind === 'seeded') { + expect(result.childCount).toBe(2); + expect(result.rootSubIssueIds).toEqual(['A']); + expect(result.alreadyExisted).toBe(false); + } + }); + + test('cycle → rejected, nothing persisted', async () => { + const ddb = { send: jest.fn() }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A', blockedBy: ['B'] }, { id: 'B', blockedBy: ['A'] }]) }), + }); + expect(result.kind).toBe('rejected'); + if (result.kind === 'rejected') { + expect(result.reason).toBe('cycle'); + expect(result.message).toMatch(/cycle/i); + } + expect(ddb.send).not.toHaveBeenCalled(); + }); + + test('Linear fetch error → error (does NOT fall back to single_task)', async () => { + const ddb = { send: jest.fn() }; + const result = await discoverOrchestration({ + ...base, ddb: ddb as never, graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: errorFetch() }), + }); + expect(result.kind).toBe('error'); + expect(ddb.send).not.toHaveBeenCalled(); + }); + + test('persistence throw → error', async () => { + const ddb = { send: jest.fn().mockResolvedValueOnce({ Item: undefined }).mockRejectedValueOnce(new Error('DDB down')) }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }]) }), + }); + expect(result.kind).toBe('error'); + }); + + test('an id collision reports a DIFFERENT message than a transient write failure', async () => { + // A collision recurs on every attempt, so the reply must not invite a + // re-trigger the way a failed write does — that would loop the user through + // something that cannot succeed. + const ddb = { + send: jest.fn().mockResolvedValueOnce({ + Item: { + sub_issue_id: '#meta', + parent_issue_ref: 'A-DIFFERENT-EPIC', + credentials_ref: 'ANOTHER-TENANT', + }, + }), + }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }]) }), + }); + + expect(result.kind).toBe('error'); + const message = (result as { message: string }).message; + expect(message).toMatch(/different issue or workspace/i); + expect(message).not.toMatch(/re-apply the trigger/i); + }); + + test('re-trigger of an existing epic with the SAME graph → extended, no new nodes', async () => { + // seedOrchestration's GetCommand sees the meta row (already seeded) → + // alreadyExisted, so discovery routes to extendOrchestration. extend's own + // loadOrchestration Query returns the existing children (A, B); the fetched + // graph is identical → no new nodes → extended with empty addedSubIssueIds. + const orchId = '#meta'; + const ddb = { + send: jest.fn() + // 1) seedOrchestration GetCommand → meta exists + .mockResolvedValueOnce({ Item: { sub_issue_id: orchId } }) + // 2) extendOrchestration loadOrchestration Query → meta + A + B + .mockResolvedValueOnce({ + Items: [ + { sub_issue_id: '#meta', orchestration_id: 'orch_x', parent_linear_issue_id: 'P', linear_workspace_id: 'WS', repo: 'o/r', platform_user_id: 'u1' }, + { sub_issue_id: 'A', orchestration_id: 'orch_x', depends_on: [], child_status: 'succeeded', parent_linear_issue_id: 'P', linear_workspace_id: 'WS', repo: 'o/r' }, + { sub_issue_id: 'B', orchestration_id: 'orch_x', depends_on: ['A'], child_status: 'succeeded', parent_linear_issue_id: 'P', linear_workspace_id: 'WS', repo: 'o/r' }, + ], + }), + }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }, { id: 'B', blockedBy: ['A'] }]) }), + }); + expect(result.kind).toBe('extended'); + if (result.kind === 'extended') expect(result.addedSubIssueIds).toEqual([]); + }); + + test('re-trigger with a NEW sub-issue → extended, adds the new node', async () => { + const ddb = { + send: jest.fn() + .mockResolvedValueOnce({ Item: { sub_issue_id: '#meta' } }) // seed: meta exists + .mockResolvedValueOnce({ + Items: [ // extend load: A + B exist + { sub_issue_id: '#meta', orchestration_id: 'orch_x', parent_linear_issue_id: 'P', linear_workspace_id: 'WS', repo: 'o/r', platform_user_id: 'u1' }, + { sub_issue_id: 'A', orchestration_id: 'orch_x', depends_on: [], child_status: 'succeeded', parent_linear_issue_id: 'P', linear_workspace_id: 'WS', repo: 'o/r' }, + { sub_issue_id: 'B', orchestration_id: 'orch_x', depends_on: ['A'], child_status: 'succeeded', parent_linear_issue_id: 'P', linear_workspace_id: 'WS', repo: 'o/r' }, + ], + }) + .mockResolvedValueOnce({}) // BatchWrite new rows + .mockResolvedValueOnce({}), // Update meta child_count + }; + // Fetched graph adds C (depends on the finished B) → C is new + releasable. + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }, { id: 'B', blockedBy: ['A'] }, { id: 'C', blockedBy: ['B'] }]) }), + }); + expect(result.kind).toBe('extended'); + if (result.kind === 'extended') { + expect(result.addedSubIssueIds).toEqual(['C']); + expect(result.releasableSubIssueIds).toEqual(['C']); // B already succeeded + } + }); + + // Trigger-agnostic seam: a custom graphSource (declarative / + // planner) drives the SAME validate→seed→reconcile pipeline, bypassing the + // Linear fetch entirely. + describe('graphSource seam (non-Linear)', () => { + test('declarative graph → seeded; never touches the Linear fetch', async () => { + const ddb = { send: jest.fn().mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}) }; + // No fetchOptions/fetchImpl — if discovery hit the Linear fetch it would throw on the real network. + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: declarativeGraphSource([ + { id: 'phase-1', depends_on: [], title: 'Plan' }, + { id: 'phase-2', depends_on: ['phase-1'], title: 'Build' }, + { id: 'phase-3', depends_on: ['phase-2'], title: 'Verify' }, + ]), + }); + expect(result.kind).toBe('seeded'); + if (result.kind === 'seeded') { + expect(result.childCount).toBe(3); + expect(result.rootSubIssueIds).toEqual(['phase-1']); // layer 0 + } + }); + + test('declarative graph still rejects an invalid DAG (cycle)', async () => { + const ddb = { send: jest.fn() }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: declarativeGraphSource([ + { id: 'x', depends_on: ['y'] }, + { id: 'y', depends_on: ['x'] }, + ]), + }); + expect(result.kind).toBe('rejected'); + expect(ddb.send).not.toHaveBeenCalled(); + }); + + test('empty declarative graph → single_task', async () => { + const ddb = { send: jest.fn() }; + const result = await discoverOrchestration({ + ...base, ddb: ddb as never, graphSource: declarativeGraphSource([]), + }); + expect(result.kind).toBe('single_task'); + expect(ddb.send).not.toHaveBeenCalled(); + }); + }); + + // Auto-integration node for fan-out. + describe('fan-out integration node', () => { + test('pure fan-out (A→B, A→C) → seeds a synthetic integration node over the leaves', async () => { + const ddb = { send: jest.fn().mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}) }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + // two leaves B, C both depending on A + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }, { id: 'B', blockedBy: ['A'] }, { id: 'C', blockedBy: ['A'] }]) }), + }); + expect(result.kind).toBe('seeded'); + if (result.kind === 'seeded') { + // A, B, C + 1 synthetic integration node + expect(result.childCount).toBe(4); + expect(result.rootSubIssueIds).toEqual(['A']); // integration node is NOT a root + } + // The BatchWrite (2nd ddb call) includes the synthetic node depending on B + C. + const puts = ddb.send.mock.calls[1][0].input.RequestItems[Object.keys(ddb.send.mock.calls[1][0].input.RequestItems)[0]] as Array<{ PutRequest: { Item: Record } }>; + const integ = puts.map((p) => p.PutRequest.Item).find((i) => String(i.sub_issue_id).endsWith('__integration')); + expect(integ).toBeDefined(); + expect([...(integ!.depends_on as string[])].sort()).toEqual(['B', 'C']); + }); + + test('linear chain → NO integration node added', async () => { + const ddb = { send: jest.fn().mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}) }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: linearGraphSource(base.accessToken, base.parentIssueRef, { fetchImpl: mockFetch([{ id: 'A' }, { id: 'B', blockedBy: ['A'] }]) }), + }); + expect(result.kind).toBe('seeded'); + if (result.kind === 'seeded') expect(result.childCount).toBe(2); // no synthetic node + }); + + test('declarative fan-out also gets an integration node', async () => { + const ddb = { send: jest.fn().mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}) }; + const result = await discoverOrchestration({ + ...base, + ddb: ddb as never, + graphSource: declarativeGraphSource([ + { id: 'x', depends_on: [] }, + { id: 'y', depends_on: [] }, + ]), + }); + expect(result.kind).toBe('seeded'); + if (result.kind === 'seeded') expect(result.childCount).toBe(3); // x, y + integration + }); + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-parent-comment.test.ts b/cdk/test/handlers/shared/orchestration-parent-comment.test.ts new file mode 100644 index 000000000..430026559 --- /dev/null +++ b/cdk/test/handlers/shared/orchestration-parent-comment.test.ts @@ -0,0 +1,348 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + type ParentCommentNode, + nodeDisplayId, + parseParentNodeReference, + renderParentDisambiguationReply, + suggestClosestNode, + looksLikeNewWork, + INTEGRATION_KEYWORD, +} from '../../../src/handlers/shared/orchestration-parent-comment'; + +const NODES: ParentCommentNode[] = [ + { sub_issue_id: 'uuid-305', linear_identifier: 'ABCA-305', title: 'Add a site-wide footer', child_task_id: 't1' }, + { sub_issue_id: 'uuid-306', linear_identifier: 'ABCA-306', title: 'Add a newsletter signup section', child_task_id: 't2' }, + { sub_issue_id: 'orch_x__integration', title: 'Integration — combine sub-issue results', child_task_id: 't3' }, +]; + +describe('parseParentNodeReference — routing a parent-epic comment to one sub-issue', () => { + test('a title keyword inside a natural sentence ("for the footer …") routes to that one sub-issue', () => { + const r = parseParentNodeReference('for the footer can you change it to "unforgettable memories await you"', NODES); + expect(r.reason).toBeNull(); + expect(r.matches).toHaveLength(1); + expect(r.matches[0].linear_identifier).toBe('ABCA-305'); + }); + + test('keyword "newsletter" → ABCA-306 only', () => { + const r = parseParentNodeReference('tweak the newsletter copy please', NODES); + expect(r.reason).toBeNull(); + expect(r.matches[0].linear_identifier).toBe('ABCA-306'); + }); + + test('Linear identifier wins outright (even alongside a keyword for another node)', () => { + const r = parseParentNodeReference('ABCA-306 also mention the footer somewhere', NODES); + expect(r.reason).toBeNull(); + expect(r.matches).toHaveLength(1); + expect(r.matches[0].linear_identifier).toBe('ABCA-306'); + }); + + test('identifier is case-insensitive', () => { + const r = parseParentNodeReference('abca-305: bump the year', NODES); + expect(r.matches[0].linear_identifier).toBe('ABCA-305'); + }); + + test('a node written under the NEUTRAL id name routes identically', () => { + // Rows persisted before and after the id was renamed coexist with no + // backfill, so reading only one name would silently drop name-the-node + // routing for half of them — the commenter types ABCA-307 and gets the + // "which sub-issue?" reply instead of their change. + const neutral: ParentCommentNode[] = [ + { sub_issue_id: 'uuid-307', display_id: 'ABCA-307', title: 'Add a hero image', child_task_id: 't4' }, + ]; + const r = parseParentNodeReference('ABCA-307: make it wider', neutral); + expect(r.reason).toBeNull(); + expect(r.matches).toHaveLength(1); + expect(nodeDisplayId(r.matches[0])).toBe('ABCA-307'); + }); + + test('nodeDisplayId is what USER-FACING text must use, under either naming', () => { + // Not only routing: the disambiguation and no-PR replies print this id back to + // the human. A legacy-only read there shows a raw UUID instead of ABCA-957 on + // any row written after the rename. + expect(nodeDisplayId({ sub_issue_id: 'u1', display_id: 'ABCA-957' })).toBe('ABCA-957'); + expect(nodeDisplayId({ sub_issue_id: 'u1', linear_identifier: 'ABCA-957' })).toBe('ABCA-957'); + // No id recorded at all → undefined, so the caller can fall back to the raw id + // deliberately rather than printing "undefined". + expect(nodeDisplayId({ sub_issue_id: 'u1' })).toBeUndefined(); + }); + + test('a MIXED epic disambiguates across both namings', () => { + // The realistic shape mid-migration: one epic extended after the rename has + // rows of both kinds, and ambiguity must be judged over all of them. + const mixed: ParentCommentNode[] = [ + { sub_issue_id: 'a', linear_identifier: 'ABCA-1', title: 'Add a pricing banner' }, + { sub_issue_id: 'b', display_id: 'ABCA-2', title: 'Add a newsletter block' }, + ]; + expect(nodeDisplayId(parseParentNodeReference('ABCA-2: reword it', mixed).matches[0])).toBe('ABCA-2'); + expect(nodeDisplayId(parseParentNodeReference('ABCA-1: reword it', mixed).matches[0])).toBe('ABCA-1'); + }); + + test('the neutral name wins when a row carries both', () => { + const both: ParentCommentNode[] = [ + { sub_issue_id: 'a', display_id: 'ABCA-9', linear_identifier: 'ABCA-9', title: 'A node' }, + ]; + expect(nodeDisplayId(both[0])).toBe('ABCA-9'); + }); + + test('no node referenced → reason "none"', () => { + const r = parseParentNodeReference('looks great, thanks!', NODES); + expect(r.reason).toBe('none'); + expect(r.matches).toHaveLength(0); + }); + + test('a keyword common to two titles → ambiguous (not a silent pick)', () => { + const nodes: ParentCommentNode[] = [ + { sub_issue_id: 'a', linear_identifier: 'ABCA-1', title: 'Add a pricing banner' }, + { sub_issue_id: 'b', linear_identifier: 'ABCA-2', title: 'Add a pricing table' }, + ]; + const r = parseParentNodeReference('update the pricing wording', nodes); + expect(r.reason).toBe('ambiguous'); + expect(r.matches).toHaveLength(2); + }); + + test('two identifiers named → ambiguous', () => { + const r = parseParentNodeReference('ABCA-305 and ABCA-306 both need the new tagline', NODES); + expect(r.reason).toBe('ambiguous'); + expect(r.matches).toHaveLength(2); + }); + + test('noise-only overlap does NOT match (e.g. "add", "page", "section")', () => { + // "add a section" shares only noise words with the titles → no match. + const r = parseParentNodeReference('please add a section somewhere', NODES); + expect(r.reason).toBe('none'); + expect(r.matches).toHaveLength(0); + }); + + test('integration node only matches on an explicit "integration"/"combined" mention', () => { + const r1 = parseParentNodeReference('check the integration result', NODES); + expect(r1.reason).toBeNull(); + expect(r1.matches[0].sub_issue_id).toBe('orch_x__integration'); + // A generic word from its title ("results") must NOT pull it in. + const r2 = parseParentNodeReference('the results look off', NODES); + expect(r2.matches.some((m) => m.sub_issue_id === 'orch_x__integration')).toBe(false); + }); + + test('"combined" targets the integration node too — it is what the panel calls it', () => { + // The panel says "Integration — combined result", "Combined PR", "Combined + // preview". A user reaches for the word they were shown, so both must work or + // the vocabulary we display is not the vocabulary we accept. + const r = parseParentNodeReference('the combined result is broken', NODES); + expect(r.reason).toBeNull(); + expect(r.matches).toHaveLength(1); + expect(r.matches[0].sub_issue_id).toBe('orch_x__integration'); + }); + + test('the keyword the reply ADVERTISES is the keyword the parser ACCEPTS', () => { + // Guards the drift that makes a helpful reply into a dead end: the + // disambiguation text tells the user to type `@bgagent integration: …`, and + // this asserts that exact word still routes. Both read the same constant, so + // this fails if either side is edited alone. + const advertised = renderParentDisambiguationReply('none', NODES); + expect(advertised).toContain(`@bgagent ${INTEGRATION_KEYWORD}:`); + const routed = parseParentNodeReference(`${INTEGRATION_KEYWORD} the dates are wrong`, NODES); + expect(routed.matches[0]?.sub_issue_id).toBe('orch_x__integration'); + }); + + test('empty / whitespace instruction → none', () => { + expect(parseParentNodeReference('', NODES).reason).toBe('none'); + expect(parseParentNodeReference(' ', NODES).reason).toBe('none'); + }); +}); + +describe('suggestClosestNode', () => { + test('returns the single best title-overlap node', () => { + // "footers" won't exact-match (plural) but "footer" stem won't either; + // use a word that overlaps a significant title word. + const s = suggestClosestNode('the newsletter box looks cramped', NODES); + expect(s?.linear_identifier).toBe('ABCA-306'); + }); + + test('returns null when nothing overlaps', () => { + expect(suggestClosestNode('ship it', NODES)).toBeNull(); + }); + + test('never suggests the integration node', () => { + const s = suggestClosestNode('the combined integration result', NODES); + expect(s).toBeNull(); // integration excluded from suggestions + }); +}); + +describe('renderParentDisambiguationReply', () => { + test('lists the real sub-issues + how to target one + the new-work path', () => { + const body = renderParentDisambiguationReply('none', NODES); + expect(body).toContain('ABCA-305 — Add a site-wide footer'); + expect(body).toContain('ABCA-306 — Add a newsletter signup section'); + expect(body).toContain('@bgagent ENG-123:'); // the how-to hint + expect(body.toLowerCase()).toContain('new work'); // the create-a-sub-issue path + expect(body).toContain('`abca` label'); + }); + + test('offers the integration node as an EXPLICIT target, with the exact phrasing', () => { + // The combined PR is the only place an integration defect can be reproduced — + // a symptom that only appears once the siblings are merged is invisible in any + // one child's PR. This reply used to hide the integration node entirely, so a + // reviewer had no way to ask for a fix there and their comment landed on + // whichever child looked closest; the backend code they were describing was + // not even present in it. + // + // The phrasing is asserted verbatim because it is an instruction the user is + // expected to copy. If it drifts from what the parser accepts, the reply tells + // people to type something that does nothing. + const body = renderParentDisambiguationReply('none', NODES); + expect(body).toContain('- Integration — combined result; target with `@bgagent integration: `'); + }); + + test('does NOT mention integration when the epic has no integration node', () => { + // A 0–1 leaf graph has none: the final child IS the combined result, so there + // is nothing separate to target and offering it would be a dead end. + const realOnly = NODES.filter((n) => !n.sub_issue_id.endsWith('__integration')); + const body = renderParentDisambiguationReply('none', realOnly); + expect(body).not.toMatch(/integration/i); + }); + + test('surfaces a "did you mean" suggestion when provided', () => { + const body = renderParentDisambiguationReply('none', NODES, NODES[0]); + expect(body).toContain('Did you mean **ABCA-305 — Add a site-wide footer**?'); + expect(body).toContain('@bgagent ABCA-305:'); + }); + + test('"Otherwise" appears ONLY when there is something to contrast with', () => { + // The word only makes sense against the "Did you mean …?" sentence, and that + // sentence is conditional — with no suggestion the reply opened by contrasting + // with nothing ("👋 I couldn't tell … Otherwise, comment on…"). + const withSuggestion = renderParentDisambiguationReply('none', NODES, NODES[0]); + expect(withSuggestion).toContain('Otherwise, comment on the specific sub-issue'); + + const without = renderParentDisambiguationReply('none', NODES); + expect(without).toContain('Comment on the specific sub-issue'); + expect(without).not.toContain('Otherwise'); + }); + + test('the commands footer does NOT bill a label re-apply as the same as retry', () => { + // Re-applying the label resolves to one of four outcomes from graph state + // (add sub-issues / retry / still running / already complete), so it is only + // equivalent in this one state — claiming equivalence taught the wrong model. + const body = renderParentDisambiguationReply('none', NODES, null, false, true); + expect(body).toContain('`@bgagent retry`'); + expect(body).toContain('keeps the ones that succeeded'); + expect(body).not.toContain('same as removing'); + }); + + test('ambiguous vs none give different lead copy', () => { + expect(renderParentDisambiguationReply('ambiguous', NODES)).toContain('more than one'); + expect(renderParentDisambiguationReply('none', NODES)).toContain("couldn't tell"); + }); + + test('the new-work flag leads with the create-a-sub-issue path', () => { + const body = renderParentDisambiguationReply('none', NODES, null, true); + expect(body).toContain('new work'); + expect(body).toContain('create a new sub-issue'); + // Leads with the new-work framing, not the generic "couldn't tell". + expect(body).not.toContain("couldn't tell"); + // Still lists the existing sub-issues for context. + expect(body).toContain('ABCA-305'); + expect(body).toContain('ABCA-306'); + }); + + // When the epic has failures, EVERY can't-act reply surfaces the `retry` + // command — so an unrecognised comment always shows what the user CAN type + // (no intent-guessing). Consistent with re-labelling. + test('hasFailures=true appends a Commands footer surfacing `@bgagent retry`', () => { + const body = renderParentDisambiguationReply('none', NODES, null, false, true); + expect(body).toContain('Commands:'); + expect(body).toContain('`@bgagent retry`'); + // Deliberately does NOT name a label re-apply as the equivalent — see the + // dedicated test below for why that framing was wrong. + expect(body).toContain('keeps the ones that succeeded'); + }); + + test('hasFailures=false omits the Commands footer (nothing to retry)', () => { + const body = renderParentDisambiguationReply('none', NODES, null, false, false); + expect(body).not.toContain('Commands:'); + expect(body).not.toContain('@bgagent retry'); + }); + + test('the retry footer also rides on the new-work path when there are failures', () => { + const body = renderParentDisambiguationReply('none', NODES, null, true, true); + expect(body).toContain('create a new sub-issue'); + expect(body).toContain('`@bgagent retry`'); + }); +}); + +describe('suggestClosestNode scores descriptions (not just titles)', () => { + // The header node's TITLE has no "blue"/"color"/"yellow"; only its + // DESCRIPTION does. The "header color to yellow instead of blue" class of + // comment should still surface it as a did-you-mean, the gap that produced a + // generic "couldn't tell" reply instead. + const DESC_NODES: ParentCommentNode[] = [ + { + sub_issue_id: 'uuid-h', + linear_identifier: 'ABCA-401', + title: 'Add a top bar with the site title', + description: 'Solid blue (#2563EB) background with white title text.', + child_task_id: 't1', + }, + { + sub_issue_id: 'uuid-f', + linear_identifier: 'ABCA-402', + title: 'Add a footer with a copyright line', + description: 'Dark background, centered copyright.', + child_task_id: 't2', + }, + ]; + + test('description word ("blue") surfaces the right node when titles miss', () => { + const s = suggestClosestNode('change the color to yellow instead of blue', DESC_NODES); + expect(s?.linear_identifier).toBe('ABCA-401'); + }); + + test('a title hit outranks a description-only hit', () => { + // "footer" is a significant TITLE word of 402; "blue" is only a DESC word of + // 401. Title weight must win. + const s = suggestClosestNode('the footer, but more blue', DESC_NODES); + expect(s?.linear_identifier).toBe('ABCA-402'); + }); + + test('still returns null when nothing overlaps title or description', () => { + expect(suggestClosestNode('ship it when ready', DESC_NODES)).toBeNull(); + }); +}); + +describe('looksLikeNewWork', () => { + test('leading additive verbs are new work', () => { + expect(looksLikeNewWork('add a testimonials section with 3 cards')).toBe(true); + expect(looksLikeNewWork('also add a pricing table')).toBe(true); + expect(looksLikeNewWork('can you create a contact form')).toBe(true); + expect(looksLikeNewWork('please build a dark mode toggle')).toBe(true); + }); + + test('change/edit verbs are NOT new work', () => { + expect(looksLikeNewWork('change the footer text to bigger')).toBe(false); + expect(looksLikeNewWork('make the colors pop more')).toBe(false); + expect(looksLikeNewWork('the footer should be centered')).toBe(false); + expect(looksLikeNewWork('ABCA-305: update the copyright')).toBe(false); + }); + + test('empty / noise instruction is not new work', () => { + expect(looksLikeNewWork('')).toBe(false); + expect(looksLikeNewWork('looks good, thanks')).toBe(false); + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-reconcile.test.ts b/cdk/test/handlers/shared/orchestration-reconcile.test.ts new file mode 100644 index 000000000..5f9439624 --- /dev/null +++ b/cdk/test/handlers/shared/orchestration-reconcile.test.ts @@ -0,0 +1,361 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + computeEpicRetryPlan, + computeReconcilePlan, + computeRecoveryPlan, + type ReconcileChild, + type TerminalOutcome, +} from '../../../src/handlers/shared/orchestration-reconcile'; +import type { ChildStatus } from '../../../src/handlers/shared/orchestration-store'; + +const row = ( + sub_issue_id: string, + child_status: ChildStatus, + depends_on: string[] = [], +): ReconcileChild => ({ sub_issue_id, depends_on, child_status }); + +/** Helper: map sub_issue_id → new status from a plan's updates. */ +function updatesById(plan: ReturnType): Record { + return Object.fromEntries(plan.statusUpdates.map((u) => [u.sub_issue_id, u.child_status])); +} + +describe('computeReconcilePlan — success releases dependents', () => { + test('A succeeds → releases its blocked dependent B', () => { + const children = [row('A', 'released'), row('B', 'blocked', ['A'])]; + const outcome: TerminalOutcome = { sub_issue_id: 'A', status: 'COMPLETED' }; + const plan = computeReconcilePlan(outcome, children); + + expect(plan.terminalSucceeded).toBe(true); + expect(updatesById(plan).A).toBe('succeeded'); + expect(plan.toRelease).toEqual(['B']); + }); + + test('linear chain: A succeeds releases B but NOT C (C still blocked on B)', () => { + const children = [ + row('A', 'released'), + row('B', 'blocked', ['A']), + row('C', 'blocked', ['B']), + ]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'COMPLETED' }, children); + expect(plan.toRelease).toEqual(['B']); + }); + + test('COMPLETED with build_passed=true is a success', () => { + const children = [row('A', 'released'), row('B', 'blocked', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'COMPLETED', build_passed: true }, children); + expect(plan.terminalSucceeded).toBe(true); + expect(plan.toRelease).toEqual(['B']); + }); + + test('build_passed undefined still counts as success (legacy records)', () => { + const children = [row('A', 'released'), row('B', 'blocked', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'COMPLETED' }, children); + expect(plan.terminalSucceeded).toBe(true); + }); +}); + +describe('computeReconcilePlan — case 1: COMPLETED but build failed', () => { + test('build_passed=false is NOT a success; dependents are skipped', () => { + const children = [row('A', 'released'), row('B', 'blocked', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'COMPLETED', build_passed: false }, children); + + expect(plan.terminalSucceeded).toBe(false); + expect(updatesById(plan).A).toBe('failed'); + expect(plan.toRelease).toEqual([]); + expect(updatesById(plan).B).toBe('skipped'); + }); +}); + +describe('computeReconcilePlan — case 2: diamond needs ALL predecessors', () => { + test('D depends on B+C; B succeeds while C still running → D NOT released', () => { + const children = [ + row('B', 'released'), + row('C', 'released'), // C's task is running, not yet succeeded + row('D', 'blocked', ['B', 'C']), + ]; + const plan = computeReconcilePlan({ sub_issue_id: 'B', status: 'COMPLETED' }, children); + expect(plan.toRelease).toEqual([]); // C hasn't succeeded yet + }); + + test('D released only once BOTH B and C have succeeded', () => { + // C is the last to finish; B already succeeded. + const children = [ + row('B', 'succeeded'), + row('C', 'released'), + row('D', 'blocked', ['B', 'C']), + ]; + const plan = computeReconcilePlan({ sub_issue_id: 'C', status: 'COMPLETED' }, children); + expect(plan.toRelease).toEqual(['D']); + }); + + test('diamond with a failed leg: C fails → D skipped even though B succeeded', () => { + const children = [ + row('B', 'succeeded'), + row('C', 'released'), + row('D', 'blocked', ['B', 'C']), + ]; + const plan = computeReconcilePlan({ sub_issue_id: 'C', status: 'FAILED' }, children); + expect(updatesById(plan).C).toBe('failed'); + expect(updatesById(plan).D).toBe('skipped'); + expect(plan.toRelease).toEqual([]); + }); +}); + +describe('computeReconcilePlan — transitive skip + sibling isolation', () => { + test('A fails → B (dep A) and C (dep B) both skipped; independent D untouched', () => { + const children = [ + row('A', 'released'), + row('B', 'blocked', ['A']), + row('C', 'blocked', ['B']), + row('D', 'blocked'), // independent root that hasn't started + ]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'FAILED' }, children); + const u = updatesById(plan); + expect(u.A).toBe('failed'); + expect(u.B).toBe('skipped'); + expect(u.C).toBe('skipped'); + expect(u.D).toBeUndefined(); // independent sibling not touched + }); + + test('CANCELLED and TIMED_OUT are failures for gating', () => { + for (const status of ['CANCELLED', 'TIMED_OUT'] as const) { + const children = [row('A', 'released'), row('B', 'blocked', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status }, children); + expect(plan.terminalSucceeded).toBe(false); + expect(updatesById(plan).B).toBe('skipped'); + } + }); + + test('does not skip a dependent that already started (released)', () => { + // B is already released (its task is running) when A fails — leave it + // to its own terminal event; do not retroactively skip. + const children = [row('A', 'released'), row('B', 'released', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'FAILED' }, children); + expect(updatesById(plan).B).toBeUndefined(); + }); +}); + +describe('computeReconcilePlan — orchestrationComplete', () => { + test('true when the last child reaches terminal', () => { + const children = [row('A', 'succeeded'), row('B', 'released', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'B', status: 'COMPLETED' }, children); + expect(plan.orchestrationComplete).toBe(true); + }); + + test('false while a released sibling is still running', () => { + const children = [ + row('A', 'released'), + row('B', 'released'), // independent, still running + ]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'COMPLETED' }, children); + expect(plan.orchestrationComplete).toBe(false); + }); + + test('true when a failure skips all remaining work', () => { + const children = [row('A', 'released'), row('B', 'blocked', ['A'])]; + const plan = computeReconcilePlan({ sub_issue_id: 'A', status: 'FAILED' }, children); + // A→failed, B→skipped → all terminal. + expect(plan.orchestrationComplete).toBe(true); + }); +}); + +/** Helper: map sub_issue_id → new status from a recovery plan's updates. */ +function recoveryUpdatesById( + plan: ReturnType, +): Record { + return Object.fromEntries(plan.statusUpdates.map((u) => [u.sub_issue_id, u.child_status])); +} + +describe('computeRecoveryPlan — comment-fixing a failed child re-releases its skipped dependents', () => { + test('BAD failed, DEP skipped → fixing BAD un-fails it + re-releases DEP', () => { + // OK succeeded, BAD failed, DEP (deps=BAD) was transitively skipped. + const children = [ + row('OK', 'succeeded'), + row('BAD', 'failed'), + row('DEP', 'skipped', ['BAD']), + ]; + const plan = computeRecoveryPlan('BAD', children); + const u = recoveryUpdatesById(plan); + expect(u.BAD).toBe('succeeded'); // un-failed + // DEP is reset skipped→blocked (the state the forward cascade understands) and, + // since BAD now succeeded, it's also in toRelease for immediate release. + expect(u.DEP).toBe('blocked'); + expect(plan.toRelease).toEqual(['DEP']); + }); + + test('no-op when the node is not currently failed (healthy iteration)', () => { + const children = [row('A', 'succeeded'), row('B', 'released', ['A'])]; + const plan = computeRecoveryPlan('A', children); + expect(plan.statusUpdates).toHaveLength(0); + expect(plan.toRelease).toHaveLength(0); + }); + + test('no-op for an unknown node id', () => { + const plan = computeRecoveryPlan('ghost', [row('A', 'failed')]); + expect(plan.statusUpdates).toHaveLength(0); + expect(plan.toRelease).toHaveLength(0); + }); + + test('a dependent with ANOTHER still-failed predecessor is reset to blocked but NOT released', () => { + // D depends on both B and C. B is being fixed, but C is still failed → + // D resets skipped→blocked (so it can release once C also recovers) but is + // NOT released now — gated the same as the original. + const children = [ + row('B', 'failed'), + row('C', 'failed'), + row('D', 'skipped', ['B', 'C']), + ]; + const plan = computeRecoveryPlan('B', children); + const u = recoveryUpdatesById(plan); + expect(u.B).toBe('succeeded'); + expect(u.D).toBe('blocked'); // reset, waiting on C + expect(plan.toRelease).toEqual([]); // C still failed → not releasable + }); + + test('diamond: fixing the apex re-releases BOTH skipped legs (predecessors satisfied)', () => { + // A(apex) failed, B & C skipped (deps=A), D skipped (deps=B,C). + const children = [ + row('A', 'failed'), + row('B', 'skipped', ['A']), + row('C', 'skipped', ['A']), + row('D', 'skipped', ['B', 'C']), + ]; + const plan = computeRecoveryPlan('A', children); + const u = recoveryUpdatesById(plan); + expect(u.A).toBe('succeeded'); + // Whole skipped subtree resets to blocked; B and C release now (A succeeded). + // D resets to blocked but does NOT release — B/C aren't succeeded yet; it + // releases later via the forward cascade when B & C land. + expect(u.B).toBe('blocked'); + expect(u.C).toBe('blocked'); + expect(u.D).toBe('blocked'); + expect(plan.toRelease.sort()).toEqual(['B', 'C']); + }); + + test('chain: fixing the head releases only the immediate next node; deeper nodes reset to blocked', () => { + // A failed → B,C skipped (B deps A, C deps B). Fixing A frees B only; C resets + // to blocked and releases later when B actually succeeds (forward cascade). + const children = [ + row('A', 'failed'), + row('B', 'skipped', ['A']), + row('C', 'skipped', ['B']), + ]; + const plan = computeRecoveryPlan('A', children); + const u = recoveryUpdatesById(plan); + expect(u.A).toBe('succeeded'); + expect(u.B).toBe('blocked'); // reset + releasable + expect(u.C).toBe('blocked'); // reset, waits for B to succeed + expect(plan.toRelease).toEqual(['B']); // only B is releasable now + }); + + test('integration node re-releases once all its (now-recovered) leaf deps succeeded', () => { + // Two leaves: GOOD succeeded, BAD failed; integration (deps GOOD,BAD) skipped. + // Fixing BAD makes both leaves succeeded → integration releases. + const children = [ + row('GOOD', 'succeeded'), + row('BAD', 'failed'), + row('INTEG', 'skipped', ['GOOD', 'BAD']), + ]; + const plan = computeRecoveryPlan('BAD', children); + const u = recoveryUpdatesById(plan); + expect(u.BAD).toBe('succeeded'); + expect(u.INTEG).toBe('blocked'); // reset; both deps now succeeded → releasable + expect(plan.toRelease).toEqual(['INTEG']); + }); +}); + +describe('computeEpicRetryPlan — re-triggering a terminal epic retries its failed/skipped children', () => { + function retryUpdatesById(plan: ReturnType): Record { + return Object.fromEntries(plan.statusUpdates.map((u) => [u.sub_issue_id, u.child_status])); + } + + test('a failed root with skipped dependents → reset them all, release the root', () => { + // 660 failed (root), 661/662 skipped (dep on 660), 663 failed (independent root), + // integration skipped (dep on all) — the shape a real terminal epic leaves behind. + const children = [ + row('660', 'failed'), + row('661', 'skipped', ['660']), + row('662', 'skipped', ['660']), + row('663', 'failed'), + row('INTEG', 'skipped', ['660', '661', '662', '663']), + ]; + const plan = computeEpicRetryPlan(children); + const u = retryUpdatesById(plan); + expect(plan.failedCount).toBe(2); + expect(plan.skippedCount).toBe(3); + expect(plan.succeededCount).toBe(0); + // Both failed roots reset to ready (no preds); skipped nodes → blocked. + expect(u['660']).toBe('ready'); + expect(u['663']).toBe('ready'); + expect(u['661']).toBe('blocked'); + expect(u['662']).toBe('blocked'); + expect(u.INTEG).toBe('blocked'); + // Only the two ready roots release now; the rest ride the forward cascade. + expect(plan.toRelease.sort()).toEqual(['660', '663']); + }); + + test('a failed node BEHIND another failed node resets to blocked (waits for the cascade), not ready', () => { + const children = [ + row('A', 'failed'), + row('B', 'failed', ['A']), // B failed AND depends on the also-failed A + ]; + const plan = computeEpicRetryPlan(children); + const u = retryUpdatesById(plan); + expect(u.A).toBe('ready'); // root, no preds + expect(u.B).toBe('blocked'); // A isn't succeeded yet → B waits + expect(plan.toRelease).toEqual(['A']); + }); + + test('succeeded nodes are NEVER touched or re-run', () => { + const children = [ + row('A', 'succeeded'), + row('B', 'failed', ['A']), + ]; + const plan = computeEpicRetryPlan(children); + const u = retryUpdatesById(plan); + expect(u.A).toBeUndefined(); // untouched + expect(u.B).toBe('ready'); // its only pred (A) already succeeded + expect(plan.toRelease).toEqual(['B']); + expect(plan.succeededCount).toBe(1); + }); + + test('nothing failed/skipped → empty plan (still-running or all-succeeded epic)', () => { + const running = computeEpicRetryPlan([row('A', 'released'), row('B', 'blocked', ['A'])]); + expect(running.statusUpdates).toEqual([]); + expect(running.toRelease).toEqual([]); + + const done = computeEpicRetryPlan([row('A', 'succeeded'), row('B', 'succeeded', ['A'])]); + expect(done.statusUpdates).toEqual([]); + expect(done.succeededCount).toBe(2); + }); + + test('idempotent: re-running the plan on the reset graph is a no-op', () => { + const children = [row('A', 'failed'), row('B', 'skipped', ['A'])]; + const first = computeEpicRetryPlan(children); + // Apply the first plan to a new graph. + const applied = children.map((c) => { + const upd = first.statusUpdates.find((u) => u.sub_issue_id === c.sub_issue_id); + return upd ? row(c.sub_issue_id, upd.child_status, c.depends_on) : c; + }); + const second = computeEpicRetryPlan(applied); + expect(second.statusUpdates).toEqual([]); // A now ready, B blocked → nothing failed/skipped + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-release.test.ts b/cdk/test/handlers/shared/orchestration-release.test.ts new file mode 100644 index 000000000..f2d023652 --- /dev/null +++ b/cdk/test/handlers/shared/orchestration-release.test.ts @@ -0,0 +1,1069 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { UpdateCommand } from '@aws-sdk/lib-dynamodb'; +import { + applyTerminalCreateFailures, + readConcurrencyBudget, + releaseChild, + releaseReadyChildren, +} from '../../../src/handlers/shared/orchestration-release'; +import { deriveOrchestrationId, type OrchestrationChildRow } from '../../../src/handlers/shared/orchestration-store'; +import { isValidIdempotencyKey } from '../../../src/handlers/shared/validation'; + +jest.mock('../../../src/handlers/shared/logger', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +const NOW = '2026-06-09T12:00:00.000Z'; + +function makeRow(overrides: Partial = {}): OrchestrationChildRow { + return { + orchestration_id: 'orch_abc', + sub_issue_id: 'SUB-1', + parent_issue_ref: 'PARENT', + credentials_ref: 'WS', + repo: 'owner/repo', + depends_on: [], + child_status: 'ready', + display_id: 'ENG-1', + title: 'Build the thing', + created_at: NOW, + updated_at: NOW, + ...overrides, + }; +} + +function created(taskId: string) { + return jest.fn().mockResolvedValue({ statusCode: 201, body: JSON.stringify({ data: { task_id: taskId } }) }); +} + +describe('releaseChild — idempotency key is accepted by the REAL validator', () => { + // Regression: the key was originally `${orchestration_id}#${sub_issue_id}`, + // but createTaskCore validates against /^[a-zA-Z0-9_-]{1,128}$/ — the '#' + // was rejected with a 400 and the child silently never started. Mocked + // createTaskCore tests didn't catch it; this asserts the generated key + // against the actual validator with production-shaped ids. + test('PARALLEL children each receive the parent\'s shared contract, verbatim', async () => { + // The incident this prevents: an epic fanned out an API child and a UI child at + // the same time. Each saw only its own sub-issue text, so each chose its own + // names — `kyoto` with `checkIn`/`checkOut` on one side, `wander-kyoto` with + // `startDate`/`endDate` on the other. Both passed their own tests, the branches + // merged without conflict, and the deployed flow was broken. Neither child was + // wrong in isolation; the agreement only ever existed in the parent. + // + // So the assertion is not "context is attached somewhere" but "BOTH siblings got + // the SAME contract text" — the property whose absence caused the divergence. + const parentContext = { + title: 'Booking flow for Kyoto trips', + description: 'Slug is `kyoto`. Dates are `checkIn`/`checkOut` (ISO-8601).', + }; + const descriptions: string[] = []; + for (const [id, title] of [['api', 'Add the booking API'], ['ui', 'Add the booking form']]) { + const createTaskCore = created(`T-${id}`); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: id, title }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + parentContext, + now: NOW, + }); + descriptions.push(createTaskCore.mock.calls[0][0].task_description as string); + } + + for (const d of descriptions) { + // Delimited and labelled, so the agent can tell the epic-wide agreement from + // its own task rather than reading one run-on prompt. + expect(d).toContain('--- SHARED ORCHESTRATION CONTEXT (the parent epic) ---'); + expect(d).toContain('--- END SHARED ORCHESTRATION CONTEXT ---'); + // The contract itself — the exact strings that diverged in the incident. + expect(d).toContain('Slug is `kyoto`'); + expect(d).toContain('`checkIn`/`checkOut`'); + // Told that siblings share it, which is what makes it binding rather than FYI. + expect(d).toMatch(/in parallel/i); + } + // And each still carries its OWN task below the shared block. + expect(descriptions[0]).toContain('Add the booking API'); + expect(descriptions[1]).toContain('Add the booking form'); + }); + + test('the shared-context preamble stays DESCRIPTIVE, so the guardrail admits the child', async () => { + // The task description is screened by a Bedrock guardrail at admission. An + // instruction-shaped preamble ("use it exactly", "do not invent an alternative") + // stacks with the epic's own imperatives and trips PROMPT_ATTACK at MEDIUM + // confidence — the child then fails with "blocked by content policy" before it + // ever runs. Live-reproduced against the deployed guardrail: the imperative + // wording blocked, this wording passed, with identical epic text after it. + // + // So this asserts the VOICE, which is the property that keeps children + // admittable. The constraint is still conveyed — by stating what the siblings + // are already working to, rather than commanding this one. + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({}), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + parentContext: { title: 'Epic', description: 'Key is `quickNote`.' }, + now: NOW, + }); + const d = createTaskCore.mock.calls[0][0].task_description as string; + const preamble = d.slice(0, d.indexOf('Epic:')); + // Second-person commands are what the filter keys on. + expect(preamble).not.toMatch(/\buse it exactly\b/i); + expect(preamble).not.toMatch(/\bdo not invent\b/i); + expect(preamble).not.toMatch(/\byou must\b/i); + // …and the constraint is still there, stated descriptively. + expect(preamble).toMatch(/in parallel/i); + expect(preamble).toMatch(/shared with/i); + }); + + test('an epic with no context does not pad the child prompt with an empty heading', async () => { + // Rows seeded before parent_context existed, and bare parents, must behave + // exactly as before rather than gaining a hollow "SHARED CONTEXT" section. + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({}), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][0].task_description).not.toContain('SHARED ORCHESTRATION CONTEXT'); + }); + + test('the integration child ALSO gets the shared contract — it verifies the boundary', async () => { + // The integration node is where a cross-boundary defect actually shows up, so + // it needs the contract most: without it the merge agent cannot tell a genuine + // mismatch from two equally-plausible conventions. + const createTaskCore = created('T-int'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'orch_abc__integration', depends_on: ['api', 'ui'] }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + parentContext: { title: 'Booking flow', description: 'Slug is `kyoto`.' }, + now: NOW, + }); + const d = createTaskCore.mock.calls[0][0].task_description as string; + expect(d).toContain('SHARED ORCHESTRATION CONTEXT'); + expect(d).toContain('Slug is `kyoto`'); + expect(d).toContain('Integrate the completed sub-issue branches'); + }); + + test('pins the coding workflow — a child must never fall back to the repo-less default', async () => { + // Every other channel pins CODING_WORKFLOW_ID at its own call site, because a + // repo-bound task with no workflow_ref resolves to the repo-OPTIONAL + // default/agent-v1, whose setup skips the stacked-base and predecessor-merge + // path entirely. + // + // A chain hides that: its child needs no merge, so the agent works from the + // base it was handed and the result looks right. A DIAMOND does not — the + // integration node is handed predecessor branches to merge, gets a clean + // default branch instead, and the agent writes a plausible approximation of + // one arm while silently dropping the other's work. Observed in production + // before this pin existed. + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({}), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + // The REQUEST body (calls[0][0]), not the context — the workflow is part of + // what is being submitted, and nothing here asserted it before. + const body = createTaskCore.mock.calls[0][0]; + expect(body.workflow_ref).toBe('coding/new-task-v1'); + }); + + test('an INTEGRATION node also pins the coding workflow — it is the case that breaks', async () => { + // The integration node is the one child that MUST merge (two predecessor + // branches), so it is the one that fails hardest on the wrong workflow. + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-int'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'orch_abc__integration', depends_on: ['SUB-1', 'SUB-2'] }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][0].workflow_ref).toBe('coding/new-task-v1'); + }); + + test('generated key passes isValidIdempotencyKey for real-world ids', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + const realRow = makeRow({ + // orch_<32 hex> — exactly what deriveOrchestrationId produces. + orchestration_id: deriveOrchestrationId('d27fcf21-4876-4be2-96c0-78099bf152de'), + // sub_issue_id is a Linear UUID in production. + sub_issue_id: 'a00650a1-4b97-46a3-9977-baede9a8f001', + }); + + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: realRow, + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + + const ctx = createTaskCore.mock.calls[0][1]; + expect(isValidIdempotencyKey(ctx.idempotencyKey)).toBe(true); + expect(ctx.idempotencyKey).not.toContain('#'); + }); + + test('key stays within the 128-char limit for max-length ids', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({ + orchestration_id: deriveOrchestrationId('x'.repeat(64)), + sub_issue_id: 'a00650a1-4b97-46a3-9977-baede9a8f001', + }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + expect(ctx.idempotencyKey.length).toBeLessThanOrEqual(128); + expect(isValidIdempotencyKey(ctx.idempotencyKey)).toBe(true); + }); +}); + +describe('releaseChild — a retry salts the idempotency key with the prior task id', () => { + test('retry=true + a prior child_task_id → key salted so a NEW task is created', async () => { + const createTaskCore = created('T-new'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ child_task_id: '01KX0WFC2DAKSEQY78ZX7WY0W4' }), // the prior FAILED task + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + retry: true, + }); + const ctx = createTaskCore.mock.calls[0][1]; + // Salted with the prior task id → distinct from the original 'orch_abc_SUB-1' + // key, so createTaskCore does NOT idempotently replay the failed task. + expect(ctx.idempotencyKey).toBe('orch_abc_SUB-1_01KX0WFC2DAKSEQY78ZX7WY0W4'); + expect(isValidIdempotencyKey(ctx.idempotencyKey)).toBe(true); + }); + + test('retry=true but NO prior task id (never-run child) → back-compat key', async () => { + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow(), // no child_task_id + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + retry: true, + }); + expect(createTaskCore.mock.calls[0][1].idempotencyKey).toBe('orch_abc_SUB-1'); + }); + + test('the key is salted whenever a prior child_task_id exists, even without retry=true', async () => { + // A child re-released while it still carries a child_task_id inherently means + // that prior task is TERMINAL (a live child is 'released', not back in + // ready/blocked). The salt must fire on the id's PRESENCE — NOT the retry + // flag — else a downstream chain child (reset to blocked, later released by + // the reconciler with retry=false) replays its dead task under the unsalted + // key. This is the fix; the OLD behavior (only salt on retry=true) was the bug. + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ child_task_id: 'OLD-TASK' }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + // NOTE: retry intentionally omitted (defaults false) — the salt still fires. + }); + expect(createTaskCore.mock.calls[0][1].idempotencyKey).toBe('orch_abc_SUB-1_OLD-TASK'); + }); + + test('first release (no prior task id) → unsalted back-compat key', async () => { + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow(), // no child_task_id + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][1].idempotencyKey).toBe('orch_abc_SUB-1'); + }); + + test('salted key stays valid + within 128 chars for production-shaped ids (uuid + ulid)', async () => { + const createTaskCore = created('T-new'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ + orchestration_id: deriveOrchestrationId('d27fcf21-4876-4be2-96c0-78099bf152de'), + sub_issue_id: 'a00650a1-4b97-46a3-9977-baede9a8f001', + child_task_id: '01KX0WFC2DAKSEQY78ZX7WY0W4', + }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + retry: true, + }); + const key = createTaskCore.mock.calls[0][1].idempotencyKey; + expect(key.length).toBeLessThanOrEqual(128); + expect(isValidIdempotencyKey(key)).toBe(true); + }); +}); + +describe('releaseChild — happy path', () => { + test('creates a task and flips the row to released', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-100'); + + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + + expect(result).toEqual({ kind: 'released', taskId: 'T-100' }); + + // createTaskCore called with linear channel + orchestration metadata + idempotency key. + const [body, ctx, requestId] = createTaskCore.mock.calls[0]; + expect(body).toMatchObject({ repo: 'owner/repo' }); + expect(body.task_description).toContain('ENG-1'); + expect(ctx).toMatchObject({ + userId: 'user-1', + channelSource: 'linear', + idempotencyKey: 'orch_abc_SUB-1', + }); + expect(ctx.channelMetadata).toMatchObject({ + orchestration_id: 'orch_abc', + orchestration_sub_issue_id: 'SUB-1', + parent_linear_issue_id: 'PARENT', + }); + expect(requestId).toBe('orch_abc_SUB-1'); + + // Flip-then-create: TWO conditional updates. + // Call 0 = the CLAIM (blocked|ready → releasing), BEFORE createTaskCore. + const claim = ddb.send.mock.calls[0][0] as UpdateCommand; + expect(claim).toBeInstanceOf(UpdateCommand); + expect(claim.input.ConditionExpression).toContain('child_status IN'); + expect(claim.input.ExpressionAttributeValues![':releasing']).toBe('releasing'); + // Call 1 = the FINALIZE (releasing → released), stamps task id + branch. + const finalize = ddb.send.mock.calls[1][0] as UpdateCommand; + expect(finalize.input.ConditionExpression).toBe('child_status = :releasing'); + expect(finalize.input.ExpressionAttributeValues![':tid']).toBe('T-100'); + expect(finalize.input.ExpressionAttributeValues![':released']).toBe('released'); + }); + + test('the planner scope (description) reaches the child task_description below the title', async () => { + const createTaskCore = created('T-desc'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ + title: 'Add a team dashboard page', + description: 'Create `dashboard.html` at the site root showing per-team stats.', + }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + const body = createTaskCore.mock.calls[0][0]; + // Title headline AND the promised deliverable both reach the agent. + expect(body.task_description).toContain('ENG-1: Add a team dashboard page'); + expect(body.task_description).toContain('dashboard.html'); + }); + + test('a description that just echoes the title is not duplicated', async () => { + const createTaskCore = created('T-echo'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ title: 'Fix the header', description: 'Fix the header' }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + const body = createTaskCore.mock.calls[0][0]; + // "Fix the header" appears once (in the "ENG-1: ..." line), not twice. + expect(body.task_description.match(/Fix the header/g)?.length).toBe(1); + }); + + test('defaults channelSource to linear when omitted (back-compat)', async () => { + const createTaskCore = created('T-def'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][1].channelSource).toBe('linear'); + }); + + test('threads an explicit channelSource onto the child task (trigger-agnostic)', async () => { + const createTaskCore = created('T-ch'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + channelSource: 'webhook', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][1].channelSource).toBe('webhook'); + }); + + test('a non-Linear child carries the neutral orchestration keys and NO Linear ids', async () => { + // The released child must not be handed ids naming rows in a workspace it has + // nothing to do with. The orchestration pair is what the reconciler maps + // back on, so that stays regardless of surface. + const createTaskCore = created('T-neutral'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ display_id: 'ENG-1' }), + platformUserId: 'user-1', + channelSource: 'webhook', + linearProjectId: 'proj-1', + linearOauthSecretArn: 'arn:secret', + linearWorkspaceSlug: 'acme', + createTaskCore: createTaskCore as never, + now: NOW, + }); + const cm = createTaskCore.mock.calls[0][1].channelMetadata as Record; + expect(cm.orchestration_id).toBeDefined(); + expect(cm.orchestration_sub_issue_id).toBeDefined(); + for (const key of [ + 'linear_workspace_id', + 'linear_issue_id', + 'linear_issue_identifier', + 'linear_project_id', + 'linear_oauth_secret_arn', + 'linear_workspace_slug', + 'parent_linear_issue_id', + ]) { + expect(cm[key]).toBeUndefined(); + } + }); + + test('a Linear child still carries every Linear key it did before', async () => { + // The gate must not quietly drop metadata the agent + orchestrator read. + const createTaskCore = created('T-linear'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ display_id: 'ENG-1' }), + platformUserId: 'user-1', + channelSource: 'linear', + linearProjectId: 'proj-1', + linearOauthSecretArn: 'arn:secret', + linearWorkspaceSlug: 'acme', + createTaskCore: createTaskCore as never, + now: NOW, + }); + const cm = createTaskCore.mock.calls[0][1].channelMetadata as Record; + expect(cm).toMatchObject({ + linear_workspace_id: 'WS', + linear_issue_identifier: 'ENG-1', + linear_project_id: 'proj-1', + linear_oauth_secret_arn: 'arn:secret', + linear_workspace_slug: 'acme', + }); + expect(cm.linear_issue_id).toBeDefined(); + expect(cm.parent_linear_issue_id).toBeDefined(); + }); + + test('threads Linear OAuth metadata when provided', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + linearOauthSecretArn: 'arn:secret', + linearWorkspaceSlug: 'acme', + linearProjectId: 'proj-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + expect(ctx.channelMetadata).toMatchObject({ + linear_oauth_secret_arn: 'arn:secret', + linear_workspace_slug: 'acme', + linear_project_id: 'proj-1', + }); + }); + + test('treats 200 idempotent replay as success', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = jest.fn().mockResolvedValue({ + statusCode: 200, + body: JSON.stringify({ data: { task_id: 'T-existing' } }), + }); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result).toEqual({ kind: 'released', taskId: 'T-existing' }); + }); +}); + +describe('releaseChild — idempotency + failure', () => { + test('ConditionalCheckFailed on the flip → already_released (no throw)', async () => { + const conditionalErr = Object.assign(new Error('conditional'), { name: 'ConditionalCheckFailedException' }); + const ddb = { send: jest.fn().mockRejectedValue(conditionalErr) }; + const createTaskCore = created('T-1'); + + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result).toEqual({ kind: 'already_released' }); + }); + + test('createTaskCore non-success → create_failed, claim rolled back to ready', async () => { + // The claim (call 0) succeeds, createTaskCore fails, so the claim + // is rolled BACK to 'ready' (call 1) — not left stranded in 'releasing'. + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = jest.fn().mockResolvedValue({ statusCode: 503, body: '{"error":{"message":"down"}}' }); + + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result.kind).toBe('create_failed'); + if (result.kind === 'create_failed') expect(result.statusCode).toBe(503); + expect(ddb.send).toHaveBeenCalledTimes(2); + const claim = ddb.send.mock.calls[0][0] as UpdateCommand; + expect(claim.input.ExpressionAttributeValues![':releasing']).toBe('releasing'); + const rollback = ddb.send.mock.calls[1][0] as UpdateCommand; + expect(rollback.input.ConditionExpression).toBe('child_status = :releasing'); + expect(rollback.input.ExpressionAttributeValues![':ready']).toBe('ready'); + }); + + test('createTaskCore throw → error, claim rolled back to ready', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = jest.fn().mockRejectedValue(new Error('boom')); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result.kind).toBe('error'); + // claim (call 0) + rollback (call 1) + expect(ddb.send).toHaveBeenCalledTimes(2); + const rollback = ddb.send.mock.calls[1][0] as UpdateCommand; + expect(rollback.input.ExpressionAttributeValues![':ready']).toBe('ready'); + }); + + // A DETERMINISTIC 4xx create failure (guardrail / validation) must NOT roll + // back to 'ready' (which loops forever every sweep, an infinite silent + // strand) — it must terminally FAIL the child so the epic settles + // finished-with-failures. + test('a guardrail 400 → create_failed_terminal, claim flipped to failed (NOT ready) + reason', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = jest.fn().mockResolvedValue({ + statusCode: 400, + body: '{"error":{"message":"Task description was blocked by content policy."}}', + }); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result.kind).toBe('create_failed_terminal'); + if (result.kind === 'create_failed_terminal') { + expect(result.statusCode).toBe(400); + // user-facing, actionable reason (guardrail is the one the user can fix) + expect(result.failureReason.toLowerCase()).toContain('content policy'); + } + // claim (call 0) + terminal-fail (call 1): flips to 'failed', not 'ready' + expect(ddb.send).toHaveBeenCalledTimes(2); + const fail = ddb.send.mock.calls[1][0] as UpdateCommand; + expect(fail.input.ConditionExpression).toBe('child_status = :releasing'); + expect(fail.input.ExpressionAttributeValues![':failed']).toBe('failed'); + expect(fail.input.ExpressionAttributeValues![':ready']).toBeUndefined(); + expect(fail.input.ExpressionAttributeValues![':reason']).toContain('content policy'); + }); + + // Against the codes createTaskCore ACTUALLY returns: 400 (validation/guardrail) + // + 422 (repo-not-onboarded) are terminal; 409 (dup) + 5xx are transient. + test('422 REPO_NOT_ONBOARDED is TERMINAL with an "onboard it" reason (sweep can\'t bound it)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = jest.fn().mockResolvedValue({ + statusCode: 422, + body: '{"error":{"code":"REPO_NOT_ONBOARDED","message":"not onboarded"}}', + }); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result.kind).toBe('create_failed_terminal'); + if (result.kind === 'create_failed_terminal') expect(result.failureReason).toMatch(/onboard/i); + const fail = ddb.send.mock.calls[1][0] as UpdateCommand; + expect(fail.input.ExpressionAttributeValues![':failed']).toBe('failed'); + }); + + test('5xx (server) + 409 (duplicate replay) are TRANSIENT — roll back to ready', async () => { + for (const statusCode of [503, 500, 409]) { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = jest.fn().mockResolvedValue({ statusCode, body: '{"error":{"message":"later"}}' }); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result.kind).toBe('create_failed'); + const rollback = ddb.send.mock.calls[1][0] as UpdateCommand; + expect(rollback.input.ExpressionAttributeValues![':ready']).toBe('ready'); + } + }); + + test('a racing releaser loses the claim (blocked|ready→releasing) and does NOT create a task', async () => { + // The core exactly-once guarantee: if the atomic claim fails + // (ConditionalCheckFailed — another releaser already claimed the row), + // releaseChild returns already_released and NEVER calls createTaskCore. + const ddb = { + send: jest.fn().mockRejectedValue( + Object.assign(new Error('claimed'), { name: 'ConditionalCheckFailedException' }), + ), + }; + const createTaskCore = created('SHOULD-NOT-RUN'); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result).toEqual({ kind: 'already_released' }); + expect(createTaskCore).not.toHaveBeenCalled(); // no double-create + expect(ddb.send).toHaveBeenCalledTimes(1); // only the failed claim + }); + + test('non-conditional DDB error on flip → error', async () => { + const ddb = { send: jest.fn().mockRejectedValue(new Error('throttle')) }; + const createTaskCore = created('T-1'); + const result = await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow(), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(result.kind).toBe('error'); + }); + + test('falls back to sub_issue_id in description when title absent', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({ title: undefined, display_id: undefined }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][0].task_description).toContain('SUB-1'); + }); +}); + +describe('releaseReadyChildren — the concurrency throttle', () => { + // 5 ready leaves, all roots (no deps) so base selection is trivial. + const readyRows = (n: number): OrchestrationChildRow[] => + Array.from({ length: n }, (_, i) => + makeRow({ sub_issue_id: `L${String(i).padStart(2, '0')}`, child_status: 'ready', depends_on: [] })); + + function createOk() { + let i = 0; + return jest.fn().mockImplementation(() => + Promise.resolve({ statusCode: 201, body: JSON.stringify({ data: { task_id: `T-${i++}` } }) })); + } + + test('undefined budget → releases ALL ready children (back-compat)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = createOk(); + const results = await releaseReadyChildren( + ddb as never, 'OrchTable', readyRows(5), { platform_user_id: 'u1' } as never, + createTaskCore as never, NOW, readyRows(5), 'main', undefined, + ); + expect(results.filter((r) => r.kind === 'released')).toHaveLength(5); + expect(createTaskCore).toHaveBeenCalledTimes(5); + }); + + test('budget caps the number released; the rest are NOT created (no fail)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = createOk(); + const rows = readyRows(5); + const results = await releaseReadyChildren( + ddb as never, 'OrchTable', rows, { platform_user_id: 'u1' } as never, + createTaskCore as never, NOW, rows, 'main', 2, // budget = 2 free slots + ); + // Only 2 tasks created — the other 3 are simply not released this pass. + expect(createTaskCore).toHaveBeenCalledTimes(2); + expect(results).toHaveLength(2); + expect(results.every((r) => r.kind === 'released')).toBe(true); + }); + + test('budget 0 → releases nothing this pass (no tasks created, no failures)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = createOk(); + const rows = readyRows(5); + const results = await releaseReadyChildren( + ddb as never, 'OrchTable', rows, { platform_user_id: 'u1' } as never, + createTaskCore as never, NOW, rows, 'main', 0, + ); + expect(createTaskCore).not.toHaveBeenCalled(); + expect(results).toHaveLength(0); + }); + + test('negative budget is treated as 0 (releases nothing)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = createOk(); + const rows = readyRows(3); + await releaseReadyChildren( + ddb as never, 'OrchTable', rows, { platform_user_id: 'u1' } as never, + createTaskCore as never, NOW, rows, 'main', -4, + ); + expect(createTaskCore).not.toHaveBeenCalled(); + }); + + test('release order is deterministic by sub_issue_id when throttled', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = createOk(); + // Shuffled input; budget 2 should pick L00, L01 (sorted), not input order. + const rows = [makeRow({ sub_issue_id: 'L02', child_status: 'ready' }), + makeRow({ sub_issue_id: 'L00', child_status: 'ready' }), + makeRow({ sub_issue_id: 'L01', child_status: 'ready' })]; + const results = await releaseReadyChildren( + ddb as never, 'OrchTable', rows, { platform_user_id: 'u1' } as never, + createTaskCore as never, NOW, rows, 'main', 2, + ); + expect(results).toHaveLength(2); + // Flip-then-create: each release issues TWO UpdateCommands + // (claim ready→releasing, then finalize releasing→released), both keyed on + // the same sub_issue_id. Dedup consecutive duplicates to get the release + // ORDER, which must be sorted L00 then L01 (not the shuffled input order). + const subsPerCall = (ddb.send.mock.calls as { 0: { input?: { Key?: { sub_issue_id?: string } } } }[]) + .map((c) => c[0]?.input?.Key?.sub_issue_id) + .filter(Boolean); + const releaseOrder = subsPerCall.filter((s, i) => s !== subsPerCall[i - 1]); + expect(releaseOrder).toEqual(['L00', 'L01']); + }); +}); + +describe('readConcurrencyBudget', () => { + test('free budget = cap - active_count', async () => { + const ddb = { send: jest.fn().mockResolvedValue({ Item: { active_count: 3 } }) }; + expect(await readConcurrencyBudget(ddb as never, 'ConcTable', 'u1', 10)).toBe(7); + }); + + test('no row yet → full cap available', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + expect(await readConcurrencyBudget(ddb as never, 'ConcTable', 'u1', 10)).toBe(10); + }); + + test('at cap → 0 (never negative)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({ Item: { active_count: 12 } }) }; + expect(await readConcurrencyBudget(ddb as never, 'ConcTable', 'u1', 10)).toBe(0); + }); + + test('read error → degrades to full cap (admission still gates)', async () => { + const ddb = { send: jest.fn().mockRejectedValue(new Error('ddb down')) }; + expect(await readConcurrencyBudget(ddb as never, 'ConcTable', 'u1', 10)).toBe(10); + }); +}); + +describe('releaseChild — parent attachments inherited by children', () => { + const ATT = [{ + attachment_id: 'a1', + type: 'file', + content_type: 'application/pdf', + filename: 'spec.pdf', + s3_key: 'attachments/user-1/epic-P/a1/spec.pdf', + s3_version_id: 'v1', + size_bytes: 42, + screening: { status: 'passed', screened_at: NOW }, + checksum_sha256: 'x'.repeat(64), + }] as never; + + test('a real feature child receives the parent preScreenedAttachments', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'uuid-A' }), + platformUserId: 'user-1', + preScreenedAttachments: ATT, + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + expect(ctx.preScreenedAttachments).toHaveLength(1); + expect(ctx.preScreenedAttachments[0].s3_key).toBe('attachments/user-1/epic-P/a1/spec.pdf'); + }); + + test('an INTEGRATION node does NOT receive attachments (pure merge, no spec needed)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'orch_abc__integration' }), + platformUserId: 'user-1', + preScreenedAttachments: ATT, + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + expect(ctx.preScreenedAttachments).toBeUndefined(); + }); + + test('no attachments provided → context omits the field (back-compat)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: ddb as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'uuid-A' }), + platformUserId: 'user-1', + createTaskCore: createTaskCore as never, + now: NOW, + }); + expect(createTaskCore.mock.calls[0][1].preScreenedAttachments).toBeUndefined(); + }); + + // A sub-issue's OWN attachments (stamped on the child row) merge + // with the inherited parent spec, de-duped, capped, own-first. + const ownRec = (id: string, name: string) => ({ + attachment_id: id, + type: 'file', + content_type: 'image/png', + filename: name, + s3_key: `attachments/user-1/child-SUB/${id}/${name}`, + s3_version_id: 'v1', + size_bytes: 10, + screening: { status: 'passed', screened_at: NOW }, + checksum_sha256: 'y'.repeat(64), + }); + + test('a child with its OWN attachment receives parent spec + own (merged, own first)', async () => { + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'uuid-A', pre_screened_attachments: [ownRec('own1', 'mock.png')] as never }), + platformUserId: 'user-1', + preScreenedAttachments: ATT, // parent spec (a1) + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + expect(ctx.preScreenedAttachments).toHaveLength(2); + // own first, then inherited parent + expect(ctx.preScreenedAttachments.map((r: { attachment_id: string }) => r.attachment_id)).toEqual(['own1', 'a1']); + }); + + test('a file on BOTH parent and child is de-duped by attachment_id', async () => { + const createTaskCore = created('T-1'); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + // child's own list re-declares a1 (same id as the parent spec) + row: makeRow({ sub_issue_id: 'uuid-A', pre_screened_attachments: [ownRec('a1', 'dup.pdf')] as never }), + platformUserId: 'user-1', + preScreenedAttachments: ATT, + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + expect(ctx.preScreenedAttachments).toHaveLength(1); + expect(ctx.preScreenedAttachments[0].attachment_id).toBe('a1'); + }); + + test('merged set over the per-task cap is trimmed to 10, dropping parent-spec files first', async () => { + const createTaskCore = created('T-1'); + const own = Array.from({ length: 8 }, (_, i) => ownRec(`own${i}`, `own${i}.png`)); + const parent = Array.from({ length: 5 }, (_, i) => ({ ...ownRec(`par${i}`, `par${i}.pdf`), attachment_id: `par${i}` })); + await releaseChild({ + ddb: { send: jest.fn().mockResolvedValue({}) } as never, + tableName: 'OrchestrationTable', + row: makeRow({ sub_issue_id: 'uuid-A', pre_screened_attachments: own as never }), + platformUserId: 'user-1', + preScreenedAttachments: parent as never, + createTaskCore: createTaskCore as never, + now: NOW, + }); + const ctx = createTaskCore.mock.calls[0][1]; + // 8 own + 5 parent = 13 → capped at 10; all 8 own survive, only 2 parent. + expect(ctx.preScreenedAttachments).toHaveLength(10); + const ids = ctx.preScreenedAttachments.map((r: { attachment_id: string }) => r.attachment_id); + expect(ids.filter((i: string) => i.startsWith('own'))).toHaveLength(8); + expect(ids.filter((i: string) => i.startsWith('par'))).toHaveLength(2); + }); +}); + +describe('applyTerminalCreateFailures — a child that never became a task', () => { + const ORCH = 'orch_1'; + /** A child row, minimal but with the fields the planner reads. */ + const row = (id: string, status: string, deps: string[] = []): OrchestrationChildRow => ({ + orchestration_id: ORCH, + sub_issue_id: id, + parent_issue_ref: 'PARENT', + credentials_ref: 'WS', + repo: 'o/r', + depends_on: deps, + child_status: status as OrchestrationChildRow['child_status'], + created_at: NOW, + updated_at: NOW, + }); + + test('skips the failed root\'s dependents and reports them as terminal', async () => { + // Why this exists at all: a guardrail rejection means no task, so no task event + // will ever wake the reconciler for this node. Nothing else would skip B or let + // the epic reach all-terminal — the panel would sit at "🔄 n/m" until a sweep. + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const children = [row('A', 'releasing'), row('B', 'blocked', ['A'])]; + + const out = await applyTerminalCreateFailures( + ddb as never, 'OrchestrationTable', ORCH, children, + [{ kind: 'create_failed_terminal', subIssueId: 'A', statusCode: 400, failureReason: 'Blocked by content policy' }] as never, + NOW, + ); + + expect(out.find((c) => c.sub_issue_id === 'A')).toMatchObject({ + child_status: 'failed', failure_reason: 'Blocked by content policy', + }); + expect(out.find((c) => c.sub_issue_id === 'B')?.child_status).toBe('skipped'); + // The skip is PERSISTED, not just patched in memory. + const writes = ddb.send.mock.calls.map((c) => c[0] as UpdateCommand) + .filter((cmd) => (cmd.input.ExpressionAttributeValues as Record)?.[':s'] === 'skipped'); + expect(writes).toHaveLength(1); + expect((writes[0].input.Key as { sub_issue_id: string }).sub_issue_id).toBe('B'); + }); + + test('a skip write can never stamp a LIVE sibling terminal', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + await applyTerminalCreateFailures( + ddb as never, 'OrchestrationTable', ORCH, + [row('A', 'releasing'), row('B', 'blocked', ['A'])], + [{ kind: 'create_failed_terminal', subIssueId: 'A', statusCode: 400, failureReason: 'r' }] as never, + NOW, + ); + const cmd = ddb.send.mock.calls[0][0] as UpdateCommand; + // Guarded on a non-live source, so a skip racing a re-release cannot mark a + // released/running child terminal and let the epic claim its rollup early. + expect(cmd.input.ConditionExpression).toBe('child_status IN (:blocked, :ready)'); + }); + + test('returns the SAME array when nothing failed terminally (callers key on identity)', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const children = [row('A', 'released')]; + + const out = await applyTerminalCreateFailures( + ddb as never, 'OrchestrationTable', ORCH, children, + [{ kind: 'released', subIssueId: 'A' }] as never, NOW, + ); + + expect(out).toBe(children); // identity: the seed path uses this to detect "nothing to settle" + expect(ddb.send).not.toHaveBeenCalled(); + }); + + test('a diamond whose two failed roots share a leaf skips that leaf once', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + const out = await applyTerminalCreateFailures( + ddb as never, 'OrchestrationTable', ORCH, + [row('A', 'releasing'), row('B', 'releasing'), row('C', 'blocked', ['A', 'B'])], + [ + { kind: 'create_failed_terminal', subIssueId: 'A', statusCode: 400, failureReason: 'r' }, + { kind: 'create_failed_terminal', subIssueId: 'B', statusCode: 400, failureReason: 'r' }, + ] as never, + NOW, + ); + expect(out.find((c) => c.sub_issue_id === 'C')?.child_status).toBe('skipped'); + const skipWrites = ddb.send.mock.calls.map((c) => c[0] as UpdateCommand) + .filter((cmd) => (cmd.input.ExpressionAttributeValues as Record)?.[':s'] === 'skipped'); + expect(skipWrites).toHaveLength(1); + }); + + test('an already-skipped dependent is not re-written', async () => { + const ddb = { send: jest.fn().mockResolvedValue({}) }; + await applyTerminalCreateFailures( + ddb as never, 'OrchestrationTable', ORCH, + [row('A', 'releasing'), row('B', 'skipped', ['A'])], + [{ kind: 'create_failed_terminal', subIssueId: 'A', statusCode: 400, failureReason: 'r' }] as never, + NOW, + ); + expect(ddb.send).not.toHaveBeenCalled(); + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-restack.test.ts b/cdk/test/handlers/shared/orchestration-restack.test.ts new file mode 100644 index 000000000..a0b9568db --- /dev/null +++ b/cdk/test/handlers/shared/orchestration-restack.test.ts @@ -0,0 +1,159 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { planDirectRestack, planRestack } from '../../../src/handlers/shared/orchestration-restack'; +import type { OrchestrationChildRow } from '../../../src/handlers/shared/orchestration-store'; + +/** Build a child row. `started` → released with a branch; else blocked. */ +function row( + sub: string, + deps: string[] = [], + opts: { started?: boolean; status?: string } = {}, +): OrchestrationChildRow { + const started = opts.started ?? true; + return { + orchestration_id: 'orch_1', + sub_issue_id: sub, + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + depends_on: deps, + child_status: (opts.status ?? (started ? 'released' : 'blocked')) as never, + created_at: 'now', + updated_at: 'now', + ...(started && { child_task_id: `task-${sub}`, child_branch_name: `branch-${sub}` }), + }; +} + +describe('planRestack', () => { + test('linear chain A→B→C, A changes → re-stack B then C (topo order)', () => { + const steps = planRestack([row('A'), row('B', ['A']), row('C', ['B'])], 'A'); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['B', 'C']); + // B merges A's branch; C merges B's branch (both in scope). + expect(steps[0].mergeBranches).toEqual(['branch-A']); + expect(steps[1].mergeBranches).toEqual(['branch-B']); + }); + + test('the changed node itself is never re-stacked', () => { + const steps = planRestack([row('A'), row('B', ['A'])], 'A'); + expect(steps.map((s) => s.child.sub_issue_id)).not.toContain('A'); + }); + + test('only STARTED dependents are re-stacked; blocked ones are skipped', () => { + // A changed; B started (released), C still blocked. + const steps = planRestack([row('A'), row('B', ['A']), row('C', ['B'], { started: false })], 'A'); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['B']); // C will get fresh code on its first release + }); + + test('diamond A→{B,C}→D, A changes → B, C, then D (D merges both updated preds)', () => { + const steps = planRestack( + [row('A'), row('B', ['A']), row('C', ['A']), row('D', ['B', 'C'])], + 'A', + ); + const ids = steps.map((s) => s.child.sub_issue_id); + expect(ids).toContain('B'); + expect(ids).toContain('C'); + expect(ids[ids.length - 1]).toBe('D'); // D is last (depends on B + C) + const dStep = steps.find((s) => s.child.sub_issue_id === 'D')!; + expect([...dStep.mergeBranches].sort()).toEqual(['branch-B', 'branch-C']); + }); + + test('mid-chain change A→B→C→D, C changes → only D re-stacks', () => { + const steps = planRestack( + [row('A'), row('B', ['A']), row('C', ['B']), row('D', ['C'])], + 'C', + ); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['D']); + expect(steps[0].mergeBranches).toEqual(['branch-C']); + }); + + test('changed node with no dependents → empty plan', () => { + expect(planRestack([row('A'), row('B', ['A'])], 'B')).toEqual([]); + }); + + test('unknown changed node → empty plan', () => { + expect(planRestack([row('A')], 'nonexistent')).toEqual([]); + }); + + test('a re-stack with no resolvable predecessor branch is dropped', () => { + // B depends on A, but A somehow has no branch — nothing to merge. + const a = { ...row('A'), child_branch_name: undefined }; + const steps = planRestack([a, row('B', ['A'])], 'A'); + expect(steps).toEqual([]); // B's only predecessor (A) has no branch → no merge → dropped + }); +}); + +describe('planDirectRestack (reconciler cascade — one hop)', () => { + test('linear chain A→B→C, A changes → re-stacks ONLY B (its direct dependent)', () => { + // C is NOT re-stacked now — it cascades when B's restack task completes. + const steps = planDirectRestack([row('A'), row('B', ['A']), row('C', ['B'])], 'A'); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['B']); + expect(steps[0].mergeBranches).toEqual(['branch-A']); + }); + + test('next hop: B changes → re-stacks ONLY C', () => { + const steps = planDirectRestack([row('A'), row('B', ['A']), row('C', ['B'])], 'B'); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['C']); + expect(steps[0].mergeBranches).toEqual(['branch-B']); + }); + + test('diamond A→{B,C}→D, A changes → re-stacks B and C (both direct), NOT D', () => { + const steps = planDirectRestack( + [row('A'), row('B', ['A']), row('C', ['A']), row('D', ['B', 'C'])], 'A', + ); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['B', 'C']); + }); + + test('diamond fan-in: B changes → D re-stacks merging BOTH arms (B + C current branches)', () => { + const steps = planDirectRestack( + [row('A'), row('B', ['A']), row('C', ['A']), row('D', ['B', 'C'])], 'B', + ); + expect(steps.map((s) => s.child.sub_issue_id)).toEqual(['D']); + expect([...steps[0].mergeBranches].sort()).toEqual(['branch-B', 'branch-C']); + }); + + test('changed node itself is never in the plan', () => { + const steps = planDirectRestack([row('A'), row('B', ['A'])], 'A'); + expect(steps.map((s) => s.child.sub_issue_id)).not.toContain('A'); + }); + + test('only STARTED direct dependents are re-stacked', () => { + const steps = planDirectRestack([row('A'), row('B', ['A'], { started: false })], 'A'); + expect(steps).toEqual([]); // B not started → gets fresh code on its first release + }); + + test('changed node with no dependents → empty', () => { + expect(planDirectRestack([row('A'), row('B', ['A'])], 'B')).toEqual([]); + }); + + test('unknown changed node → empty', () => { + expect(planDirectRestack([row('A')], 'nope')).toEqual([]); + }); + + test('direct dependent whose every predecessor lacks a branch is dropped', () => { + const a = { ...row('A'), child_branch_name: undefined }; + expect(planDirectRestack([a, row('B', ['A'])], 'A')).toEqual([]); + }); + + test('does NOT recurse: grandchild is untouched even when started', () => { + // A→B→C all started; A changes → only B (C waits for B to finish). + const steps = planDirectRestack([row('A'), row('B', ['A']), row('C', ['B'])], 'A'); + expect(steps.map((s) => s.child.sub_issue_id)).not.toContain('C'); + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-rollup.test.ts b/cdk/test/handlers/shared/orchestration-rollup.test.ts new file mode 100644 index 000000000..fea2e3893 --- /dev/null +++ b/cdk/test/handlers/shared/orchestration-rollup.test.ts @@ -0,0 +1,844 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +const slackFetchMock = jest.fn(); +const slackFetchTsMock = jest.fn(); +jest.mock('../../../src/handlers/shared/slack-api', () => ({ + slackFetch: (...a: unknown[]) => slackFetchMock(...a), + slackFetchTs: (...a: unknown[]) => slackFetchTsMock(...a), +})); +jest.mock('../../../src/handlers/shared/slack-verify', () => ({ + SLACK_SECRET_PREFIX: 'bgagent/slack/', + getSlackSecret: async () => 'xoxb-token', +})); + +const loggerMock = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; +jest.mock('../../../src/handlers/shared/logger', () => ({ logger: loggerMock })); + +import type { IssueRef } from '../../../src/handlers/shared/orchestration-channel'; +import { channelForSource, registerChannelFactory } from '../../../src/handlers/shared/orchestration-channel-factory'; +import { makeSlackChannel, slackThreadRef } from '../../../src/handlers/shared/orchestration-channel-slack'; +import { ORCH_LOG } from '../../../src/handlers/shared/orchestration-log-events'; +import { + renderRollupComment, + renderStatusBlock, + renderEpicPanel, + buildPanelRows, + truncateQuote, + cascadeNodeLabel, + rollupKindFromChildren, + postRollup, + upsertEpicPanel, + type RollupChildView, + type EpicPanelRow, +} from '../../../src/handlers/shared/orchestration-rollup'; +import type { OrchestrationChildRow } from '../../../src/handlers/shared/orchestration-store'; +import { DEFAULT_LABEL_FILTER } from '../../../src/handlers/shared/trigger-label'; + +/** + * A stand-in surface adapter. The rollup is exercised through the Channel it + * actually calls, so these tests assert the neutral operations (and the + * capability guards) rather than any one surface's API. + */ +type FakeChannel = ReturnType; + +function makeFakeChannel() { + return { + kind: 'linear' as const, + postComment: jest.fn, unknown[]>() + .mockResolvedValue({ commentId: '' }), + upsertComment: jest.fn, unknown[]>() + .mockResolvedValue({ commentId: 'cmt-1' }), + reportFailure: jest.fn, unknown[]>().mockResolvedValue(undefined), + transitionState: jest.fn, unknown[]>().mockResolvedValue(true), + replaceIssueReaction: jest.fn, unknown[]>().mockResolvedValue(true), + }; +} + +const parent: IssueRef = { issueId: 'PARENT', credentialsRef: 'WS' }; +let channel: FakeChannel; + +const view = (sub: string, status: string, ident?: string, title?: string, pr_url?: string): RollupChildView => ({ + sub_issue_id: sub, + child_status: status, + ...(ident && { display_id: ident }), + ...(title && { title }), + ...(pr_url && { pr_url }), +}); + +describe('renderRollupComment', () => { + test('complete: all succeeded → completion heading + counts', () => { + const body = renderRollupComment('complete', [ + view('a', 'succeeded', 'ENG-1', 'Step A'), + view('b', 'succeeded', 'ENG-2', 'Step B'), + ]); + expect(body).toContain('orchestration complete'); + expect(body).toContain('2 succeeded, 0 failed, 0 skipped'); + expect(body).toContain('✅ ENG-1: Step A'); + }); + + test('partial_failure: shows failed + skipped with icons + summary', () => { + const body = renderRollupComment('partial_failure', [ + view('a', 'failed', 'ENG-1'), + view('b', 'skipped', 'ENG-2'), + view('c', 'succeeded', 'ENG-3'), + ]); + expect(body).toContain('finished with failures'); + expect(body).toContain('1 succeeded, 1 failed, 1 skipped'); + expect(body).toContain('❌ ENG-1'); + expect(body).toContain('⏭️ ENG-2'); + }); + + test('cancelled: cancellation heading', () => { + const body = renderRollupComment('cancelled', [view('a', 'failed', 'ENG-1')]); + expect(body).toContain('cancelled'); + }); + + test('children are sorted by identifier (deterministic comment)', () => { + const body = renderRollupComment('complete', [ + view('z', 'succeeded', 'ENG-9'), + view('a', 'succeeded', 'ENG-1'), + ]); + expect(body.indexOf('ENG-1')).toBeLessThan(body.indexOf('ENG-9')); + }); + + // Per-child PR links + integration-node combined-PR callout. + test('renders a PR link on a child line when pr_url is present', () => { + const body = renderRollupComment('complete', [ + view('a', 'succeeded', 'ENG-1', 'Step A', 'https://github.com/o/r/pull/10'), + view('b', 'succeeded', 'ENG-2', 'Step B'), // no PR + ]); + expect(body).toContain('✅ ENG-1: Step A — succeeded — [PR](https://github.com/o/r/pull/10)'); + // A child without a PR renders no link (no broken markdown). + expect(body).toContain('✅ ENG-2: Step B — succeeded'); + expect(body).not.toContain('ENG-2: Step B — succeeded — [PR]'); + }); + + test('surfaces the integration node combined PR as a prominent callout', () => { + const body = renderRollupComment('complete', [ + view('a', 'succeeded', 'ENG-1', 'Leaf A', 'https://github.com/o/r/pull/1'), + view('b', 'succeeded', 'ENG-2', 'Leaf B', 'https://github.com/o/r/pull/2'), + view('orch_x__integration', 'succeeded', undefined, 'Integration — combine sub-issue results', 'https://github.com/o/r/pull/9'), + ]); + expect(body).toContain('🔗 **Combined PR (all sub-issues merged):** [https://github.com/o/r/pull/9](https://github.com/o/r/pull/9)'); + // The callout appears BEFORE the per-child list. + expect(body.indexOf('Combined PR')).toBeLessThan(body.indexOf('ENG-1')); + }); + + test('no combined-PR callout when the integration node opened no PR', () => { + const body = renderRollupComment('partial_failure', [ + view('a', 'succeeded', 'ENG-1', 'Leaf A', 'https://github.com/o/r/pull/1'), + view('orch_x__integration', 'skipped', undefined, 'Integration — combine sub-issue results'), // no PR (skipped) + ]); + expect(body).not.toContain('Combined PR'); + }); + + test('no combined-PR callout for a plain chain (no integration node)', () => { + const body = renderRollupComment('complete', [ + view('a', 'succeeded', 'ENG-1', 'A', 'https://github.com/o/r/pull/1'), + view('b', 'succeeded', 'ENG-2', 'B', 'https://github.com/o/r/pull/2'), + ]); + expect(body).not.toContain('Combined PR'); + }); +}); + +describe('renderStatusBlock — the live status block', () => { + test('header shows N/M complete (terminal children only)', () => { + const body = renderStatusBlock([ + view('a', 'succeeded', 'ENG-1', 'Guide'), + view('b', 'released', 'ENG-2', 'Cards'), + view('c', 'blocked', 'ENG-3', 'Quiz'), + ]); + expect(body).toContain('1/3 complete'); + expect(body).toContain('🔄 **ABCA orchestration**'); + }); + + test('maps in-flight statuses to human words (running / blocked)', () => { + const body = renderStatusBlock([ + view('a', 'released', 'ENG-1', 'A'), + view('b', 'blocked', 'ENG-2', 'B'), + ]); + expect(body).toContain('ENG-1: A — running'); + expect(body).toContain('ENG-2: B — blocked'); + }); + + test('links a child PR in the live block when pr_url is known', () => { + const body = renderStatusBlock([ + view('a', 'released', 'ENG-1', 'A', 'https://github.com/o/r/pull/7'), + view('b', 'blocked', 'ENG-2', 'B'), + ]); + expect(body).toContain('ENG-1: A — running — [PR](https://github.com/o/r/pull/7)'); + expect(body).toContain('ENG-2: B — blocked'); + expect(body).not.toContain('ENG-2: B — blocked — [PR]'); + }); + + test('terminal statuses keep their word + icon', () => { + const body = renderStatusBlock([ + view('a', 'succeeded', 'ENG-1'), + view('b', 'failed', 'ENG-2'), + view('c', 'skipped', 'ENG-3'), + ]); + expect(body).toContain('✅ ENG-1 — succeeded'); + expect(body).toContain('❌ ENG-2 — failed'); + expect(body).toContain('⏭️ ENG-3 — skipped'); + expect(body).toContain('3/3 complete'); + }); + + test('children sorted by identifier (stable edit-in-place body)', () => { + const body = renderStatusBlock([view('z', 'released', 'ENG-9'), view('a', 'released', 'ENG-1')]); + expect(body.indexOf('ENG-1')).toBeLessThan(body.indexOf('ENG-9')); + }); +}); + +describe('rollupKindFromChildren', () => { + test('all succeeded → complete', () => { + expect(rollupKindFromChildren([view('a', 'succeeded'), view('b', 'succeeded')])).toBe('complete'); + }); + test('any failed → partial_failure', () => { + expect(rollupKindFromChildren([view('a', 'succeeded'), view('b', 'failed')])).toBe('partial_failure'); + }); + test('any skipped → partial_failure', () => { + expect(rollupKindFromChildren([view('a', 'succeeded'), view('b', 'skipped')])).toBe('partial_failure'); + }); +}); + +const row = (sub: string, status: string): OrchestrationChildRow => ({ + orchestration_id: 'orch_1', + sub_issue_id: sub, + parent_linear_issue_id: 'PARENT', + linear_workspace_id: 'WS', + repo: 'o/r', + depends_on: [], + child_status: status as never, + created_at: 'now', + updated_at: 'now', +}); + +describe('upsertEpicPanel — the maturing panel + parent-state mirror', () => { + beforeEach(() => { + slackFetchMock.mockReset().mockResolvedValue(true); + slackFetchTsMock.mockReset().mockResolvedValue('1700000000.002'); + channel = makeFakeChannel(); + loggerMock.info.mockReset(); + loggerMock.warn.mockReset(); + }); + + test('edits the existing panel comment in place when given its id', async () => { + const id = await upsertEpicPanel({ + channel, parent, statusCommentId: 'panel-1', children: [row('a', 'running')], + }); + expect(id).toBe('cmt-1'); + const [, , existing] = channel.upsertComment.mock.calls[0]; + expect(existing).toEqual({ commentId: 'panel-1' }); + }); + + test('a surface that returns no usable comment id reports no panel id', async () => { + // A blank id must not be persisted — the next edit would address a comment + // that doesn't exist. "No id" is the honest answer. + channel.upsertComment.mockResolvedValue({ commentId: '' }); + expect(await upsertEpicPanel({ channel, parent, children: [row('a', 'running')] })).toBeNull(); + }); + + test('in progress → re-opens the parent to running (regression allowed) + 👀', async () => { + // A settled epic sits in awaiting-review; re-opening moves backward WITHIN + // the same state category, which the adapter refuses unless asked. Without + // the opt-in the re-open is silently dropped and the epic reads finished + // while children are running. + await upsertEpicPanel({ + channel, parent, children: [row('a', 'running')], inProgress: true, mirrorParentState: true, + }); + expect(channel.transitionState).toHaveBeenCalledWith(parent, 'started', { allowRegression: true }); + expect(channel.replaceIssueReaction).toHaveBeenCalledWith(parent, 'started'); + }); + + test('all succeeded → advances to awaiting-review + ✅', async () => { + await upsertEpicPanel({ + channel, parent, children: [row('a', 'succeeded')], inProgress: false, mirrorParentState: true, + }); + expect(channel.transitionState).toHaveBeenCalledWith(parent, 'in_review'); + expect(channel.replaceIssueReaction).toHaveBeenCalledWith(parent, 'succeeded'); + }); + + test('finished with failures → leaves the state, marks ❌', async () => { + await upsertEpicPanel({ + channel, + parent, + children: [row('a', 'succeeded'), row('b', 'failed')], + inProgress: false, + mirrorParentState: true, + }); + expect(channel.transitionState).not.toHaveBeenCalled(); + expect(channel.replaceIssueReaction).toHaveBeenCalledWith(parent, 'failed'); + }); + + test('mirrorParentState: false edits the panel only — no state or reaction', async () => { + await upsertEpicPanel({ + channel, parent, children: [row('a', 'succeeded')], inProgress: false, mirrorParentState: false, + }); + expect(channel.upsertComment).toHaveBeenCalled(); + expect(channel.transitionState).not.toHaveBeenCalled(); + expect(channel.replaceIssueReaction).not.toHaveBeenCalled(); + }); + + test('the REAL Slack adapter drives the epic panel, with its omitted ops skipped', async () => { + // The engine claim under test: an unmodified rollup drives a genuinely + // different surface. Slack omits transitionState/revertState (no workflow + // state), so the capability guards must skip them and still deliver the + // panel — proving the abstraction on a real adapter, not a fake. + const slack = makeSlackChannel(); + const id = await upsertEpicPanel({ + channel: slack, + parent: { issueId: slackThreadRef('C1', '1700000000.001'), credentialsRef: 'T1' }, + children: [row('a', 'succeeded')], + inProgress: false, + mirrorParentState: true, // asks for a state mirror Slack cannot do + }); + // The panel landed... + expect(id).toBe('1700000000.002'); + expect(slackFetchTsMock).toHaveBeenCalled(); + // ...the ✅ marker went on the thread root... + const adds = slackFetchMock.mock.calls.filter((c) => c[1] === 'reactions.add'); + expect((adds[0][2] as { name: string }).name).toBe('white_check_mark'); + // ...and NOTHING attempted a state transition (there is no such Slack method). + const methods = [ + ...slackFetchMock.mock.calls.map((c) => c[1]), + ...slackFetchTsMock.mock.calls.map((c) => c[1]), + ]; + expect(methods.every((m) => typeof m === 'string' && m.startsWith('chat.') || String(m).startsWith('reactions.'))).toBe(true); + }); + + test('a DOWNSTREAM surface the core never imports drives the panel end to end', async () => { + // The extensibility claim, exercised rather than asserted about: this surface + // exists only in this test, is resolved through the same registry lookup the + // reconciler uses, and reaches the engine with no edit to the interface, the + // lookup, or the rollup. Slack proves the interface fits a non-tracker, but it + // ships in-tree — so it cannot show that nothing in the core has to know. + const posted: Array<{ issue: string; body: string }> = []; + const unregister = registerChannelFactory('acme-tracker', (registryTable) => ({ + kind: 'acme-tracker', + postComment: async () => null, + upsertComment: async (issue, body) => { + // The per-surface credentials table reached the adapter, so a panel is + // addressed with this tenant's credentials and not another surface's. + expect(registryTable).toBe('AcmeRegistry'); + posted.push({ issue: issue.issueId, body }); + return { commentId: 'acme-panel-1' }; + }, + reportFailure: async () => undefined, + // Reactions supported; workflow state deliberately NOT — a third capability + // shape, different from both Linear (all of it) and Slack (none of it). + replaceIssueReaction: async () => true, + })); + try { + const channelFromRegistry = channelForSource('acme-tracker', { 'acme-tracker': 'AcmeRegistry' }); + expect(channelFromRegistry).toBeDefined(); + + const id = await upsertEpicPanel({ + channel: channelFromRegistry!, + parent: { issueId: 'acme-epic-1', credentialsRef: 'acme-tenant-1' }, + children: [row('a', 'succeeded')], + inProgress: false, + mirrorParentState: true, // asks for a transition this surface cannot do + }); + + expect(id).toBe('acme-panel-1'); + expect(posted).toHaveLength(1); + expect(posted[0].issue).toBe('acme-epic-1'); + // The panel body is the engine's own rendering — the surface supplied none of it. + expect(posted[0].body).toContain('ABCA orchestration complete'); + } finally { + unregister(); + } + }); + + test('a surface without reactions or transitions still gets its panel', async () => { + const commentOnly = makeFakeChannel(); + delete (commentOnly as Partial).transitionState; + delete (commentOnly as Partial).replaceIssueReaction; + const id = await upsertEpicPanel({ + channel: commentOnly, parent, children: [row('a', 'succeeded')], mirrorParentState: true, + }); + expect(id).toBe('cmt-1'); + }); + + test('a panel-comment failure is swallowed and reported as no id', async () => { + channel.upsertComment.mockRejectedValue(new Error('surface hiccup')); + expect(await upsertEpicPanel({ channel, parent, children: [row('a', 'running')] })).toBeNull(); + expect(loggerMock.warn).toHaveBeenCalled(); + }); + + test('a mirror failure does not lose the panel id already obtained', async () => { + channel.transitionState.mockRejectedValue(new Error('states query timed out')); + const id = await upsertEpicPanel({ + channel, parent, children: [row('a', 'succeeded')], inProgress: false, mirrorParentState: true, + }); + expect(id).toBe('cmt-1'); + }); +}); + +describe('postRollup', () => { + beforeEach(() => { + channel = makeFakeChannel(); + loggerMock.info.mockReset(); + loggerMock.warn.mockReset(); + }); + + test('success → posts comment + logs orch.rollup.posted', async () => { + const ok = await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + }); + expect(ok).toBe(true); + expect(channel.postComment).toHaveBeenCalledTimes(1); + // The stable log event automated tests grep for. + const posted = loggerMock.info.mock.calls.find((c) => c[1]?.event === ORCH_LOG.rollupPosted); + expect(posted).toBeDefined(); + expect(posted![1]).toMatchObject({ orchestration_id: 'orch_1', parent_issue_id: 'PARENT', rollup_kind: 'complete' }); + }); + + test('complete → advances parent to awaiting-review + ✅ reaction (mirrors children)', async () => { + await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + }); + expect(channel.transitionState).toHaveBeenCalledWith(parent, 'in_review'); + expect(channel.replaceIssueReaction).toHaveBeenCalledWith(parent, 'succeeded'); + }); + + test('partial_failure → does NOT advance state, swaps to ❌ reaction', async () => { + await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'partial_failure', + children: [row('a', 'failed')], + }); + expect(channel.transitionState).not.toHaveBeenCalled(); + expect(channel.replaceIssueReaction).toHaveBeenCalledWith(parent, 'failed'); + }); + + test('comment fails → does NOT transition state or react (state mirrors only on posted rollup)', async () => { + channel.postComment.mockResolvedValue(null); + await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + }); + expect(channel.transitionState).not.toHaveBeenCalled(); + expect(channel.replaceIssueReaction).not.toHaveBeenCalled(); + }); + + test('post returns false → logs orch.rollup.failed, returns false', async () => { + channel.postComment.mockResolvedValue(null); + const ok = await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'partial_failure', + children: [row('a', 'failed')], + }); + expect(ok).toBe(false); + expect(loggerMock.warn.mock.calls.some((c) => c[1]?.event === ORCH_LOG.rollupFailed)).toBe(true); + }); + + test('a surface without reactions or transitions still posts the rollup', async () => { + // The capability guards must skip the mirror rather than throw, so a + // comment-only surface gets the rollup comment and nothing else. + const commentOnly = makeFakeChannel(); + delete (commentOnly as Partial).transitionState; + delete (commentOnly as Partial).replaceIssueReaction; + const ok = await postRollup({ + channel: commentOnly, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + }); + expect(ok).toBe(true); + expect(commentOnly.postComment).toHaveBeenCalledTimes(1); + }); + + test('with statusCommentId → EDITS the live block in place (no fresh comment)', async () => { + const ok = await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + statusCommentId: 'cmt-1', + }); + expect(ok).toBe(true); + // Edited the existing comment; did NOT post a fresh one. + expect(channel.upsertComment).toHaveBeenCalledWith(parent, expect.any(String), { commentId: 'cmt-1' }); + expect(channel.postComment).not.toHaveBeenCalled(); + }); + + test('threads prUrls → rendered comment links child PRs + combined PR', async () => { + await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded'), row('orch_1__integration', 'succeeded')], + prUrls: { + a: 'https://github.com/o/r/pull/3', + orch_1__integration: 'https://github.com/o/r/pull/9', + }, + }); + const body = channel.postComment.mock.calls[0][1] as string; + expect(body).toContain('[PR](https://github.com/o/r/pull/3)'); + expect(body).toContain('🔗 **Combined PR (all sub-issues merged):**'); + expect(body).toContain('https://github.com/o/r/pull/9'); + }); + + test('without statusCommentId → posts a fresh comment', async () => { + await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + }); + expect(channel.postComment).toHaveBeenCalledTimes(1); + expect(channel.upsertComment).not.toHaveBeenCalled(); + }); + + test('post throws → swallowed, logs orch.rollup.failed, returns false', async () => { + channel.postComment.mockRejectedValue(new Error('surface down')); + const ok = await postRollup({ + channel, + parent, + orchestrationId: 'orch_1', + kind: 'complete', + children: [row('a', 'succeeded')], + }); + expect(ok).toBe(false); + expect(loggerMock.warn.mock.calls.some((c) => c[1]?.event === ORCH_LOG.rollupFailed)).toBe(true); + }); +}); + +describe('truncateQuote', () => { + test('short text passes through, trimmed + whitespace-collapsed', () => { + expect(truncateQuote(' the button doesnt work ')).toBe('the button doesnt work'); + }); + test('long text is truncated with an ellipsis', () => { + const out = truncateQuote('a'.repeat(60), 40); + expect(out.length).toBe(40); + expect(out.endsWith('…')).toBe(true); + }); +}); + +describe('cascadeNodeLabel — the short name used inside a cascade reason', () => { + test('integration node → "the integration" (not its raw synthetic title)', () => { + // The raw integration-node title read clumsily + // in the possessive reason "Integration — combine sub-issue results's change". + const label = cascadeNodeLabel('orch_abc__integration', undefined, 'Integration — combine sub-issue results'); + expect(label).toBe('the integration'); + // Reads cleanly in the possessive: "the integration's change". + expect(`updating to include ${label}'s change`).toBe("updating to include the integration's change"); + }); + + test('real node prefers the Linear identifier', () => { + expect(cascadeNodeLabel('uuid-1', 'ABCA-42', 'Some title')).toBe('ABCA-42'); + }); + + test('real node with no identifier falls back to title, then a generic name', () => { + expect(cascadeNodeLabel('uuid-1', undefined, 'Some title')).toBe('Some title'); + expect(cascadeNodeLabel('uuid-1')).toBe('a predecessor'); + }); +}); + +describe('renderEpicPanel — the single maturing panel', () => { + // Named for its shape: an EpicPanelRow (what the panel renders), distinct from + // the file-level `row`, which builds an OrchestrationChildRow (what is stored). + const panelRow = (sub: string, status: string, opts: Partial = {}): EpicPanelRow => ({ + sub_issue_id: sub, child_status: status, ...opts, + }); + + test('in-progress header shows N/M complete', () => { + const body = renderEpicPanel({ + inProgress: true, + rows: [ + panelRow('a', 'succeeded', { display_id: 'ENG-1', title: 'A' }), + panelRow('b', 'released', { display_id: 'ENG-2', title: 'B' }), + panelRow('c', 'blocked', { display_id: 'ENG-3', title: 'C' }), + ], + }); + expect(body).toContain('🔄 **ABCA orchestration** · 1/3 complete'); + expect(body).toContain('✅ ENG-1: A — succeeded'); + expect(body).toContain('🔄 ENG-2: B — running'); + expect(body).toContain('⏳ ENG-3: C — blocked'); + }); + + test('all settled + ok → complete header; failures → ⚠️', () => { + expect(renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'succeeded')] })) + .toContain('✅ **ABCA orchestration complete**'); + expect(renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'succeeded'), panelRow('b', 'failed')] })) + .toContain('⚠️ **ABCA orchestration finished with failures**'); + }); + + // A SETTLED-with-failures panel tells the user how to retry. Not shown while + // in-flight or on a clean complete. + test('the retry hint leads with the comment and offers the label only as a fallback', () => { + // Presenting the two as equivalent was wrong: re-applying the label is a + // gesture with four possible meanings (add sub-issues / retry / still running / + // already complete) resolved from graph state, so on an epic that gained a + // sub-issue AND has a failure it does strictly more than a retry. + const body = renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'succeeded'), panelRow('b', 'failed')] }); + expect(body).toContain('To retry:'); + expect(body).toContain('`@bgagent retry`'); + // The label is still discoverable — just not billed as the same thing. It + // must be the label that actually TRIGGERS: this assertion previously pinned + // a hardcoded project-specific label, so the hint told users to re-apply + // something the webhook does not filter on and nothing happened. + expect(body).toContain('re-applying the `bgagent` label'); + expect(body).not.toContain('either way'); + // The comment must be named before the label, so the reliable route reads first. + expect(body.indexOf('`@bgagent retry`')).toBeLessThan(body.indexOf('re-applying')); + }); + + test('the retry hint names the project\'s own trigger label, not a hardcoded one', () => { + // The trigger label is per-project configurable, so a hint that hardcodes one + // is wrong for every project that renamed it — the user follows the + // instruction and nothing fires. + const body = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded'), panelRow('b', 'failed')], + labelFilter: 'ship', + }); + expect(body).toContain('re-applying the `ship` label'); + expect(body).not.toContain('`bgagent` label'); + }); + + test('the retry hint falls back to the platform default label when none is supplied', () => { + const body = renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'succeeded'), panelRow('b', 'failed')] }); + expect(body).toContain(`re-applying the \`${DEFAULT_LABEL_FILTER}\` label`); + }); + + test('no retry hint on a clean complete, or while still in progress', () => { + expect(renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'succeeded')] })) + .not.toContain('To retry:'); + // in-flight, even with a not-yet-terminal failed sibling shown: no hint until settled. + expect(renderEpicPanel({ inProgress: true, rows: [panelRow('a', 'released'), panelRow('b', 'failed')] })) + .not.toContain('To retry:'); + }); + + test('PR link shown ONLY when a PR exists (first run mid-flight has none)', () => { + const body = renderEpicPanel({ + inProgress: true, + rows: [ + panelRow('a', 'released', { display_id: 'ENG-1', title: 'A' }), // running, no PR yet + panelRow('b', 'succeeded', { display_id: 'ENG-2', title: 'B', pr_url: 'https://github.com/o/r/pull/9' }), + ], + }); + expect(body).toContain('🔄 ENG-1: A — running\n'); // no — [PR] suffix + expect(body).not.toContain('ENG-1: A — running — [PR]'); + expect(body).toContain('✅ ENG-2: B — succeeded — [PR](https://github.com/o/r/pull/9)'); + }); + + test('a row with updatingReason renders 🔄 updating , even when status is succeeded', () => { + const body = renderEpicPanel({ + inProgress: true, + rows: [ + panelRow('a', 'succeeded', { + display_id: 'ENG-1', + title: 'UI', + pr_url: 'https://github.com/o/r/pull/7', + updatingReason: 'per ENG-2\'s "button doesnt work"', + }), + ], + }); + expect(body).toContain('🔄 ENG-1: UI — updating per ENG-2\'s "button doesnt work" — [PR](https://github.com/o/r/pull/7)'); + }); + + test('a mid-update row keeps the header in-progress (does NOT count as done)', () => { + // inProgress is passed true by the caller when any row is updating; the + // updating row is excluded from the done count. + const body = renderEpicPanel({ + inProgress: true, + rows: [ + panelRow('a', 'succeeded', { updatingReason: 'to include ENG-3\'s change' }), + panelRow('b', 'succeeded'), + ], + }); + expect(body).toContain('· 1/2 complete'); // only b counts as done + }); + + test('integration node renders friendly, never its raw id', () => { + const body = renderEpicPanel({ + inProgress: false, + rows: [ + panelRow('a', 'succeeded', { display_id: 'ENG-1' }), + panelRow('orch_x__integration', 'succeeded', { pr_url: 'https://github.com/o/r/pull/9' }), + ], + combinedPrUrl: 'https://github.com/o/r/pull/9', + }); + expect(body).toContain('Integration — combined result'); + expect(body).not.toContain('orch_x__integration'); + expect(body).toContain('🔗 **Combined PR (all sub-issues merged):**'); + }); + + test('a failed row renders an indented diagnostic sub-line (what failed + where to read it)', () => { + const reason = 'Combined build failed after merging the sub-issue branches — see the build log in CloudWatch for task `t-int`.'; + const body = renderEpicPanel({ + inProgress: false, + rows: [ + panelRow('a', 'succeeded', { display_id: 'ENG-1' }), + panelRow('orch_x__integration', 'failed', { failureReason: reason }), + ], + }); + // The integration row + its sub-line on the very next line (indented ↳). + expect(body).toContain(`- ❌ Integration — combined result — failed\n ↳ ${reason}`); + }); + + test('the sub-line is ONLY rendered for failed rows (not succeeded/skipped/running)', () => { + const reason = 'should not appear'; + const succeeded = renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'succeeded', { failureReason: reason })] }); + expect(succeeded).not.toContain('↳'); + expect(succeeded).not.toContain(reason); + // A skipped row (predecessor failed) gets no sub-line either — only the + // node that actually failed carries the diagnostic. + const skipped = renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'skipped', { failureReason: reason })] }); + expect(skipped).not.toContain('↳'); + }); + + test('a failed row with NO reason resolved still renders cleanly (no dangling ↳)', () => { + const body = renderEpicPanel({ inProgress: false, rows: [panelRow('a', 'failed', { display_id: 'ENG-1' })] }); + expect(body).toContain('❌ ENG-1 — failed'); + expect(body).not.toContain('↳'); + }); + + test('embeds the preview screenshot when present', () => { + // No combinedPrUrl → no integration node merged anything, so this is the + // final node's own preview and must NOT be labelled "combined". + const body = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded')], + combinedScreenshotUrl: 'https://cdn/x.png', + }); + expect(body).toContain('🖼️ **Preview**'); + expect(body).toContain('![preview](https://cdn/x.png)'); + }); + + test('labels it "Combined preview" only when an integration node merged the leaves', () => { + // The combined PR callout is the signal that several leaves were merged. + // Calling a chain's final-node preview "combined" would tell a reviewer + // branches were merged when none were. + const combined = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded')], + combinedPrUrl: 'https://github.com/o/r/pull/99', + combinedScreenshotUrl: 'https://cdn/x.png', + }); + expect(combined).toContain('🖼️ **Combined preview**'); + const chain = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded')], + combinedScreenshotUrl: 'https://cdn/x.png', + }); + expect(chain).not.toContain('Combined preview'); + }); + + test('makes the combined preview a clickable deep-link when the preview URL is known', () => { + const body = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded')], + combinedScreenshotUrl: 'https://cdn/x.png', + combinedPreviewUrl: 'https://my-app-abc123.vercel.app', + }); + expect(body).toContain('🖼️ **Preview**'); + // Linked image: the embedded screenshot opens the running site. + expect(body).toContain('[![preview](https://cdn/x.png)](https://my-app-abc123.vercel.app)'); + // Plain "open it" link too, for clients that don't render linked images. + expect(body).toContain('[Open the preview](https://my-app-abc123.vercel.app)'); + }); + + test('percent-encodes parens in the preview URL so it cannot break out of the markdown link', () => { + const body = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded')], + combinedScreenshotUrl: 'https://cdn/x.png', + combinedPreviewUrl: 'https://preview.vercel.app/x)](https://evil/a.png)', + }); + // No raw `](` breakout delimiter from the attacker-controlled preview URL. + expect(body).not.toContain('x)](https://evil'); + expect(body).toContain('%29'); // encoded paren survives + }); + + test('falls back to a plain embedded image when no preview URL is known', () => { + const body = renderEpicPanel({ + inProgress: false, + rows: [panelRow('a', 'succeeded')], + combinedScreenshotUrl: 'https://cdn/x.png', + }); + expect(body).toContain('![preview](https://cdn/x.png)'); + expect(body).not.toContain('[![preview]'); // not a linked image + expect(body).not.toContain('Open the preview'); + }); + + test('rows are sorted by identifier for a stable edited body', () => { + const body = renderEpicPanel({ + inProgress: true, + rows: [ + panelRow('z', 'released', { display_id: 'ENG-9' }), + panelRow('a', 'released', { display_id: 'ENG-1' }), + ], + }); + expect(body.indexOf('ENG-1')).toBeLessThan(body.indexOf('ENG-9')); + }); +}); + +describe('buildPanelRows — the failureReasons map lands on row.failureReason', () => { + const child = (sub: string, status: string): OrchestrationChildRow => ({ + orchestration_id: 'orch_1', + sub_issue_id: sub, + parent_linear_issue_id: 'parent', + linear_workspace_id: 'ws', + repo: 'o/r', + depends_on: [], + child_status: status as OrchestrationChildRow['child_status'], + created_at: 'now', + updated_at: 'now', + }); + + test('attaches the reason to the matching failed row, and only that row', () => { + const rows = buildPanelRows( + [child('a', 'succeeded'), child('orch_1__integration', 'failed')], + {}, + {}, + { orch_1__integration: 'Combined build failed — see CloudWatch for task `t-int`.' }, + ); + expect(rows.find((r) => r.sub_issue_id === 'a')?.failureReason).toBeUndefined(); + expect(rows.find((r) => r.sub_issue_id === 'orch_1__integration')?.failureReason) + .toMatch(/Combined build failed/); + }); + + test('omits failureReason when no map is supplied (back-compat)', () => { + const rows = buildPanelRows([child('a', 'failed')]); + expect(rows[0].failureReason).toBeUndefined(); + }); +}); diff --git a/cdk/test/handlers/shared/orchestration-store.test.ts b/cdk/test/handlers/shared/orchestration-store.test.ts index da2750c80..db571c1ad 100644 --- a/cdk/test/handlers/shared/orchestration-store.test.ts +++ b/cdk/test/handlers/shared/orchestration-store.test.ts @@ -253,6 +253,56 @@ describe('seedOrchestration — first write', () => { }); }); +describe("seedOrchestration — the project's trigger label round-trips", () => { + test('a supplied trigger_label is persisted on the meta row and read back', async () => { + // The epic panel's retry hint must name the label that actually FIRES, and + // that label is per-project configurable. Seed time is the only point where + // the project mapping is in hand — the reconciler works from this row and has + // no project id to look one up with — so if it is not persisted here the panel + // can only ever show the platform default, which is wrong for any project that + // renamed its label. + const ddb = makeDdb(); + ddb.send.mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}); + + await seedOrchestration({ + ddb: ddb as never, + tableName: TABLE, + parentIssueRef: 'PARENT', + credentialsRef: 'WS', + repo: 'o/r', + children: [child('A')], + now: NOW, + releaseContext: { ...RC, trigger_label: 'ship' }, + }); + + const batch = ddb.send.mock.calls[1][0]; + const puts = batch.input.RequestItems[TABLE] as Array<{ PutRequest: { Item: Record } }>; + const meta = puts.map((p) => p.PutRequest.Item).find((i) => i.sub_issue_id === '#meta'); + expect(meta).toBeDefined(); + expect(meta!.trigger_label).toBe('ship'); + }); + + test('an absent trigger_label writes no attribute (older rows load fine)', async () => { + const ddb = makeDdb(); + ddb.send.mockResolvedValueOnce({ Item: undefined }).mockResolvedValueOnce({}); + await seedOrchestration({ + ddb: ddb as never, + tableName: TABLE, + parentIssueRef: 'PARENT', + credentialsRef: 'WS', + repo: 'o/r', + children: [child('A')], + now: NOW, + releaseContext: RC, + }); + const batch = ddb.send.mock.calls[1][0]; + const puts = batch.input.RequestItems[TABLE] as Array<{ PutRequest: { Item: Record } }>; + const meta = puts.map((p) => p.PutRequest.Item).find((i) => i.sub_issue_id === '#meta'); + expect(meta).toBeDefined(); + expect('trigger_label' in meta!).toBe(false); + }); +}); + describe('seedOrchestration — idempotent replay', () => { test('skips writing when a meta row already exists', async () => { const ddb = makeDdb(); @@ -562,6 +612,70 @@ describe('renamed row attributes stay readable across the rename', () => { }); }); +describe("loadOrchestration — the project's trigger label is hydrated", () => { + test('a persisted trigger_label reaches release_context so the panel can name it', async () => { + // Persisting it is only half the job: if the load path drops the attribute the + // panel still falls back to the platform default and the retry hint still names + // a label that may not fire. + const ddb = { + send: jest.fn().mockResolvedValue({ + Items: [ + { + orchestration_id: 'orch_1', + sub_issue_id: '#meta', + repo: 'o/r', + platform_user_id: 'u1', + child_count: 1, + parent_issue_ref: 'P', + credentials_ref: 'WS', + trigger_label: 'ship', + }, + { + orchestration_id: 'orch_1', + sub_issue_id: 'uuid-A', + depends_on: [], + child_status: 'succeeded', + parent_issue_ref: 'P', + credentials_ref: 'WS', + repo: 'o/r', + }, + ], + }), + }; + const snap = await loadOrchestration(ddb as never, TABLE, 'orch_1'); + expect(snap!.meta.release_context.trigger_label).toBe('ship'); + }); + + test('an absent trigger_label leaves release_context without one (older rows)', async () => { + const ddb = { + send: jest.fn().mockResolvedValue({ + Items: [ + { + orchestration_id: 'orch_1', + sub_issue_id: '#meta', + repo: 'o/r', + platform_user_id: 'u1', + child_count: 1, + parent_issue_ref: 'P', + credentials_ref: 'WS', + }, + { + orchestration_id: 'orch_1', + sub_issue_id: 'uuid-A', + depends_on: [], + child_status: 'succeeded', + parent_issue_ref: 'P', + credentials_ref: 'WS', + repo: 'o/r', + }, + ], + }), + }; + const snap = await loadOrchestration(ddb as never, TABLE, 'orch_1'); + expect(snap!.meta.release_context.trigger_label).toBeUndefined(); + }); +}); + describe('loadOrchestration — dedup marker rows are not children', () => { test('excludes ack# marker rows from children (only real sub-issues count)', async () => { const ddb = {