fix: MCP tool discovery pipeline, agentTool migration, and e2e reorganization

.gitignore

@@ -10,6 +10,9 @@ build/
 *.local
 *.tsbuildinfo
 
+# Context dumps (may contain secrets)
+.context
+
 # Environment variables
 .env
 .env.local
@@ -18,8 +21,7 @@ docker/.env
 .env.development.local
 .env.test.local
 .env.production.local
-.env.eng-104
-.env.eng-104
+.env.e2e
 .shipsec-instance
 
 # Logs

backend/scripts/generate-openapi.ts

@@ -14,11 +14,9 @@ async function generateOpenApi() {
 
   const { AppModule } = await import('../src/app.module');
 
-  console.log('Creating Nest app...');
   const app = await NestFactory.create(AppModule, {
     logger: ['error', 'warn'],
   });
-  console.log('Nest app created');
 
   // Set global prefix to match production
   app.setGlobalPrefix('api/v1');
@@ -31,7 +29,6 @@ async function generateOpenApi() {
     .build();
 
   const document = SwaggerModule.createDocument(app, config);
-  console.log('Document paths keys:', Object.keys(document.paths));
   const cleaned = cleanupOpenApiDoc(document);
   const repoRootSpecPath = join(__dirname, '..', '..', 'openapi.json');
   const payload = JSON.stringify(cleaned, null, 2);

backend/src/components/components.controller.ts

@@ -6,6 +6,7 @@ import '@shipsec/studio-worker/components';
 import {
   componentRegistry,
   extractPorts,
+  isAgentCallable,
   getToolSchema,
   type CachedComponentMetadata,
 } from '@shipsec/component-sdk';
@@ -46,8 +47,8 @@ function serializeComponent(entry: CachedComponentMetadata) {
     outputs: entry.outputs ?? [],
     parameters: entry.parameters ?? [],
     examples: metadata.examples ?? [],
-    agentTool: metadata.agentTool ?? null,
-    toolSchema: metadata.agentTool?.enabled ? getToolSchema(component) : null,
+    toolProvider: component.toolProvider ?? null,
+    toolSchema: isAgentCallable(component) ? getToolSchema(component) : null,
   };
 }
 
@@ -224,13 +225,13 @@ export class ComponentsController {
             type: 'array',
             items: { type: 'string' },
           },
-          agentTool: {
+          toolProvider: {
             type: 'object',
             nullable: true,
             properties: {
-              enabled: { type: 'boolean' },
-              toolName: { type: 'string', nullable: true },
-              toolDescription: { type: 'string', nullable: true },
+              kind: { type: 'string', enum: ['component', 'mcp-server', 'mcp-group'] },
+              name: { type: 'string' },
+              description: { type: 'string' },
             },
           },
         },

backend/src/database/schema/mcp-servers.ts

@@ -27,7 +27,7 @@ export const mcpGroups = pgTable(
     // Credential configuration
     credentialContractName: varchar('credential_contract_name', { length: 191 }).notNull(),
     credentialMapping: jsonb('credential_mapping')
-      .$type<Record<string, unknown> | null>()
+      .$type<Record<string, string> | null>()
       .default(null),
 
     // Default Docker image for servers in this group

backend/src/mcp-groups/dto/mcp-groups.dto.ts

@@ -11,7 +11,7 @@ export const McpGroupSchema = z.object({
   name: z.string(),
   description: z.string().nullable().optional(),
   credentialContractName: z.string(),
-  credentialMapping: z.record(z.string(), z.unknown()).nullable().optional(),
+  credentialMapping: z.record(z.string(), z.string()).nullable().optional(),
   defaultDockerImage: z.string().nullable().optional(),
   enabled: z.boolean(),
   createdAt: z.string().datetime(),
@@ -43,7 +43,7 @@ export const CreateMcpGroupSchema = z.object({
   name: z.string().min(1),
   description: z.string().nullable().optional(),
   credentialContractName: z.string().min(1),
-  credentialMapping: z.record(z.string(), z.unknown()).nullable().optional(),
+  credentialMapping: z.record(z.string(), z.string()).nullable().optional(),
   defaultDockerImage: z.string().nullable().optional(),
   enabled: z.boolean().optional(),
 });
@@ -54,7 +54,7 @@ export const UpdateMcpGroupSchema = z.object({
   name: z.string().min(1).optional(),
   description: z.string().nullable().optional(),
   credentialContractName: z.string().min(1).optional(),
-  credentialMapping: z.record(z.string(), z.unknown()).nullable().optional(),
+  credentialMapping: z.record(z.string(), z.string()).nullable().optional(),
   defaultDockerImage: z.string().nullable().optional(),
   enabled: z.boolean().optional(),
 });
@@ -86,7 +86,7 @@ export const McpGroupResponseSchema = z.object({
   name: z.string(),
   description: z.string().nullable(),
   credentialContractName: z.string(),
-  credentialMapping: z.record(z.string(), z.unknown()).nullable(),
+  credentialMapping: z.record(z.string(), z.string()).nullable(),
   defaultDockerImage: z.string().nullable(),
   enabled: z.boolean(),
   templateHash: z.string().nullable().optional(),
@@ -208,7 +208,7 @@ export const GroupTemplateSchema = z.object({
   name: z.string().min(1),
   description: z.string().optional(),
   credentialContractName: z.string().min(1),
-  credentialMapping: z.record(z.string(), z.unknown()).optional(),
+  credentialMapping: z.record(z.string(), z.string()).optional(),
   defaultDockerImage: z.string().min(1),
   version: TemplateVersionSchema,
   servers: z.array(GroupTemplateServerSchema),

backend/src/mcp-groups/mcp-group-templates.ts

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
  * Server configuration within a group template
  */
 export interface GroupTemplateServer {
+  id?: string;
   name: string;
   description?: string;
   transportType: 'http' | 'stdio' | 'sse' | 'websocket';
@@ -33,7 +34,7 @@ export interface McpGroupTemplate {
   name: string;
   description?: string;
   credentialContractName: string;
-  credentialMapping?: Record<string, unknown>;
+  credentialMapping?: Record<string, string>;
   defaultDockerImage: string;
   version: TemplateVersion;
   servers: GroupTemplateServer[];
@@ -52,6 +53,7 @@ export function computeTemplateHash(template: McpGroupTemplate): string {
     defaultDockerImage: template.defaultDockerImage,
     version: template.version,
     servers: template.servers.map((s) => ({
+      id: s.id,
       name: s.name,
       description: s.description,
       transportType: s.transportType,
@@ -76,16 +78,26 @@ const __dirname = dirname(__filename);
 const TEMPLATE_DIR = join(__dirname, 'templates');
 
 function loadTemplates(): Record<string, McpGroupTemplate> {
-  const templates: Record<string, McpGroupTemplate> = {};
-  const files = readdirSync(TEMPLATE_DIR).filter((file) => file.endsWith('.json'));
+  try {
+    const templates: Record<string, McpGroupTemplate> = {};
+    const files = readdirSync(TEMPLATE_DIR).filter((file) => file.endsWith('.json'));
 
-  for (const file of files) {
-    const raw = JSON.parse(readFileSync(join(TEMPLATE_DIR, file), 'utf-8')) as McpGroupTemplate;
-    const slug = raw.slug || file.replace(/\.json$/, '');
-    templates[slug] = { ...raw, slug };
-  }
+    for (const file of files) {
+      try {
+        const raw = JSON.parse(readFileSync(join(TEMPLATE_DIR, file), 'utf-8')) as McpGroupTemplate;
 
-  return templates;
+        const slug = raw.slug || file.replace(/\.json$/, '');
+        templates[slug] = { ...raw, slug };
+      } catch (fileError) {
+        console.error(`[loadTemplates] ERROR loading ${file}:`, fileError);
+        throw fileError;
+      }
+    }
+    return templates;
+  } catch (e) {
+    console.error('[loadTemplates] FATAL ERROR:', e);
+    throw e;
+  }
 }
 
 /**

backend/src/mcp-groups/mcp-groups-seeding.service.ts

@@ -11,11 +11,7 @@ import {
   computeTemplateHash,
   type McpGroupTemplate,
 } from './mcp-group-templates';
-import {
-  SyncTemplatesResponse,
-  GroupTemplateDto,
-  GroupTemplateServerDto,
-} from './dto/mcp-groups.dto';
+import { SyncTemplatesResponse, GroupTemplateDto } from './dto/mcp-groups.dto';
 
 /**
  * Result of syncing a single template
@@ -52,7 +48,21 @@ export class McpGroupsSeedingService {
    * Get all available templates as DTOs
    */
   getAllTemplates(): GroupTemplateDto[] {
-    return Object.values(MCP_GROUP_TEMPLATES).map((template) => this.templateToDto(template));
+    try {
+      this.logger.log(
+        '[getAllTemplates] Starting, templates count:',
+        Object.keys(MCP_GROUP_TEMPLATES).length,
+      );
+      const result = Object.values(MCP_GROUP_TEMPLATES).map((template) => {
+        this.logger.log('[getAllTemplates] Converting template:', template.slug);
+        return this.templateToDto(template);
+      });
+      this.logger.log('[getAllTemplates] Successfully converted', result.length, 'templates');
+      return result;
+    } catch (e) {
+      this.logger.error('[getAllTemplates] ERROR:', e);
+      throw e;
+    }
   }
 
   /**
@@ -365,16 +375,17 @@ export class McpGroupsSeedingService {
     dto.version = template.version;
     dto.templateHash = computeTemplateHash(template);
     dto.servers = template.servers.map((server) => {
-      const serverDto = new GroupTemplateServerDto();
-      serverDto.name = server.name;
-      serverDto.description = server.description;
-      serverDto.transportType = server.transportType;
-      serverDto.endpoint = server.endpoint;
-      serverDto.command = server.command;
-      serverDto.args = server.args;
-      serverDto.recommended = server.recommended ?? false;
-      serverDto.defaultSelected = server.defaultSelected ?? true;
-      return serverDto;
+      return {
+        id: server.id,
+        name: server.name,
+        description: server.description,
+        transportType: server.transportType,
+        endpoint: server.endpoint,
+        command: server.command,
+        args: server.args,
+        recommended: server.recommended ?? false,
+        defaultSelected: server.defaultSelected ?? true,
+      };
     });
     return dto;
   }

backend/src/mcp-groups/mcp-groups.repository.ts

@@ -20,7 +20,7 @@ export interface McpGroupUpdateData {
   name?: string;
   description?: string | null;
   credentialContractName?: string;
-  credentialMapping?: Record<string, unknown> | null;
+  credentialMapping?: Record<string, string> | null;
   defaultDockerImage?: string | null;
   enabled?: boolean;
 }

backend/src/mcp-groups/mcp-groups.service.ts

@@ -354,4 +354,39 @@ export class McpGroupsService implements OnModuleInit {
       toolCount: cached.toolCount,
     };
   }
+
+  /**
+   * Get server configuration for a group template server
+   * Used by MCP group runtime to fetch server details
+   */
+  async getServerConfig(
+    groupSlug: string,
+    serverId: string,
+  ): Promise<{ command: string; args?: string[]; endpoint?: string }> {
+    const template = this.seedingService.getTemplateBySlug(groupSlug);
+    if (!template) {
+      throw new BadRequestException(`MCP group template '${groupSlug}' not found`);
+    }
+
+    // Search for server by ID (primary) or name (fallback)
+    const server = template.servers.find((s: any) => s.id === serverId || s.name === serverId);
+    if (!server) {
+      throw new BadRequestException(`Server '${serverId}' not found in group '${groupSlug}'`);
+    }
+
+    // Return server configuration
+    const config: { command: string; args?: string[]; endpoint?: string } = {
+      command: server.command || '',
+    };
+
+    if (server.args && server.args.length > 0) {
+      config.args = server.args;
+    }
+
+    if (server.endpoint) {
+      config.endpoint = server.endpoint;
+    }
+
+    return config;
+  }
 }

backend/src/mcp-groups/templates/aws.json

@@ -4,10 +4,10 @@
   "description": "Essential AWS security tools for auditing, monitoring, and incident response",
   "credentialContractName": "core.credential.aws",
   "credentialMapping": {
-    "accessKeyId": "AWS_ACCESS_KEY_ID",
-    "secretAccessKey": "AWS_SECRET_ACCESS_KEY",
-    "sessionToken": "AWS_SESSION_TOKEN",
-    "region": "AWS_REGION"
+    "AWS_ACCESS_KEY_ID": "accessKeyId",
+    "AWS_SECRET_ACCESS_KEY": "secretAccessKey",
+    "AWS_SESSION_TOKEN": "sessionToken",
+    "AWS_REGION": "region"
   },
   "defaultDockerImage": "shipsec/mcp-aws-suite:latest",
   "version": {
@@ -17,6 +17,7 @@
   },
   "servers": [
     {
+      "id": "aws-cloudtrail",
       "name": "cloudtrail",
       "description": "CloudTrail auditing - event lookup, user activity analysis, compliance investigations",
       "transportType": "stdio",
@@ -25,6 +26,7 @@
       "defaultSelected": true
     },
     {
+      "id": "aws-iam",
       "name": "iam",
       "description": "IAM security - user/role management, permission analysis, access key audit",
       "transportType": "stdio",
@@ -33,6 +35,7 @@
       "defaultSelected": true
     },
     {
+      "id": "aws-s3-tables",
       "name": "s3-tables",
       "description": "S3 Tables security - S3 Tables bucket policies, access controls",
       "transportType": "stdio",
@@ -41,6 +44,7 @@
       "defaultSelected": true
     },
     {
+      "id": "aws-cloudwatch",
       "name": "cloudwatch",
       "description": "CloudWatch monitoring - logs, metrics, alarms for security events",
       "transportType": "stdio",
@@ -49,6 +53,7 @@
       "defaultSelected": true
     },
     {
+      "id": "aws-network",
       "name": "aws-network",
       "description": "AWS Network - VPC, networking configuration, security groups",
       "transportType": "stdio",
@@ -57,6 +62,7 @@
       "defaultSelected": false
     },
     {
+      "id": "aws-lambda",
       "name": "lambda",
       "description": "Lambda security - function permissions, runtime analysis, IAM roles",
       "transportType": "stdio",
@@ -65,6 +71,7 @@
       "defaultSelected": false
     },
     {
+      "id": "aws-dynamodb",
       "name": "dynamodb",
       "description": "DynamoDB security - table access policies, encryption, point-in-time recovery",
       "transportType": "stdio",
@@ -73,6 +80,7 @@
       "defaultSelected": false
     },
     {
+      "id": "aws-documentation",
       "name": "aws-documentation",
       "description": "AWS docs - real-time access to official AWS security documentation",
       "transportType": "stdio",
@@ -81,6 +89,7 @@
       "defaultSelected": false
     },
     {
+      "id": "aws-well-architected",
       "name": "well-architected-security",
       "description": "Security review - AWS Well-Architected security best practices framework",
       "transportType": "stdio",
@@ -89,6 +98,7 @@
       "defaultSelected": false
     },
     {
+      "id": "aws-api",
       "name": "aws-api",
       "description": "AWS API explorer - interact with any AWS service API directly",
       "transportType": "stdio",