merge(workflow): resolve conflict in useWorkflowImportExport

Combines port resolution and secret validation features from both branches:
- Keeps dynamic port resolution for Analytics components (from feature branch)
- Retains secret reference validation (from main)
- Merges all required imports (api, useSecretStore, useComponentStore)

Signed-off-by: Aseem Shrey <LuD1161@users.noreply.github.com>

Author: LuD1161

AGENTS.md

@@ -41,7 +41,7 @@ Local development runs as **multiple app instances** (PM2) on top of **one share
 
 - Shared infra (Docker Compose project `shipsec-infra`): Postgres/Temporal/Redpanda/Redis/MinIO/Loki on fixed ports.
 - Per-instance apps: `shipsec-{frontend,backend,worker}-N`.
-- Isolation is via per-instance DB + Temporal namespace/task queue + Kafka topic suffixing (not per-instance infra containers).
+- Isolation is via per-instance DB + Temporal namespace/task queue + Kafka topic suffixing + instance-scoped Kafka consumer groups/client IDs (not per-instance infra containers).
 - The workspace can have an **active instance** (stored in `.shipsec-instance`, gitignored).
 
 **Agent rule:** before running any dev commands, ensure you’re targeting the intended instance.

backend/src/agent-trace/__tests__/agent-trace-ingest.service.spec.ts

@@ -0,0 +1,61 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+
+import { AgentTraceIngestService } from '../agent-trace-ingest.service';
+import type { AgentTraceRepository } from '../agent-trace.repository';
+
+const ORIGINAL_ENV = { ...process.env };
+
+function restoreEnv(): void {
+  process.env = { ...ORIGINAL_ENV };
+}
+
+describe('AgentTraceIngestService', () => {
+  beforeEach(() => {
+    restoreEnv();
+    process.env.LOG_KAFKA_BROKERS = 'localhost:19092';
+    delete process.env.SHIPSEC_INSTANCE;
+    delete process.env.AGENT_TRACE_KAFKA_GROUP_ID;
+    delete process.env.AGENT_TRACE_KAFKA_CLIENT_ID;
+  });
+
+  afterEach(() => {
+    restoreEnv();
+  });
+
+  test('uses legacy defaults when SHIPSEC_INSTANCE is unset', () => {
+    const repository = { append: async () => undefined } as unknown as AgentTraceRepository;
+    const service = new AgentTraceIngestService(repository) as unknown as {
+      kafkaGroupId: string;
+      kafkaClientId: string;
+    };
+
+    expect(service.kafkaGroupId).toBe('shipsec-agent-trace-ingestor');
+    expect(service.kafkaClientId).toBe('shipsec-backend-agent-trace');
+  });
+
+  test('uses instance-scoped defaults when SHIPSEC_INSTANCE is set', () => {
+    process.env.SHIPSEC_INSTANCE = '7';
+    const repository = { append: async () => undefined } as unknown as AgentTraceRepository;
+    const service = new AgentTraceIngestService(repository) as unknown as {
+      kafkaGroupId: string;
+      kafkaClientId: string;
+    };
+
+    expect(service.kafkaGroupId).toBe('shipsec-agent-trace-ingestor-7');
+    expect(service.kafkaClientId).toBe('shipsec-backend-agent-trace-7');
+  });
+
+  test('prefers explicit env vars over defaults', () => {
+    process.env.SHIPSEC_INSTANCE = '3';
+    process.env.AGENT_TRACE_KAFKA_GROUP_ID = 'custom-agent-trace-group';
+    process.env.AGENT_TRACE_KAFKA_CLIENT_ID = 'custom-agent-trace-client';
+    const repository = { append: async () => undefined } as unknown as AgentTraceRepository;
+    const service = new AgentTraceIngestService(repository) as unknown as {
+      kafkaGroupId: string;
+      kafkaClientId: string;
+    };
+
+    expect(service.kafkaGroupId).toBe('custom-agent-trace-group');
+    expect(service.kafkaClientId).toBe('custom-agent-trace-client');
+  });
+});

backend/src/agent-trace/agent-trace-ingest.service.ts

@@ -26,9 +26,16 @@ export class AgentTraceIngestService implements OnModuleInit, OnModuleDestroy {
     // Use instance-aware topic name
     const topicResolver = getTopicResolver();
     this.kafkaTopic = topicResolver.getAgentTraceTopic();
+    const instanceId = process.env.SHIPSEC_INSTANCE;
+    const defaultGroupId = instanceId
+      ? `shipsec-agent-trace-ingestor-${instanceId}`
+      : 'shipsec-agent-trace-ingestor';
+    const defaultClientId = instanceId
+      ? `shipsec-backend-agent-trace-${instanceId}`
+      : 'shipsec-backend-agent-trace';
 
-    this.kafkaGroupId = process.env.AGENT_TRACE_KAFKA_GROUP_ID ?? 'shipsec-agent-trace-ingestor';
-    this.kafkaClientId = process.env.AGENT_TRACE_KAFKA_CLIENT_ID ?? 'shipsec-backend-agent-trace';
+    this.kafkaGroupId = process.env.AGENT_TRACE_KAFKA_GROUP_ID ?? defaultGroupId;
+    this.kafkaClientId = process.env.AGENT_TRACE_KAFKA_CLIENT_ID ?? defaultClientId;
   }
 
   async onModuleInit(): Promise<void> {

backend/src/node-io/__tests__/node-io-ingest.service.spec.ts

@@ -0,0 +1,70 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+
+import { NodeIOIngestService } from '../node-io-ingest.service';
+import type { NodeIORepository } from '../node-io.repository';
+
+const ORIGINAL_ENV = { ...process.env };
+
+function restoreEnv(): void {
+  process.env = { ...ORIGINAL_ENV };
+}
+
+describe('NodeIOIngestService', () => {
+  beforeEach(() => {
+    restoreEnv();
+    process.env.LOG_KAFKA_BROKERS = 'localhost:19092';
+    delete process.env.SHIPSEC_INSTANCE;
+    delete process.env.NODE_IO_KAFKA_GROUP_ID;
+    delete process.env.NODE_IO_KAFKA_CLIENT_ID;
+  });
+
+  afterEach(() => {
+    restoreEnv();
+  });
+
+  test('uses legacy defaults when SHIPSEC_INSTANCE is unset', () => {
+    const repository = {
+      recordStart: async () => undefined,
+      recordCompletion: async () => undefined,
+    } as unknown as NodeIORepository;
+    const service = new NodeIOIngestService(repository) as unknown as {
+      kafkaGroupId: string;
+      kafkaClientId: string;
+    };
+
+    expect(service.kafkaGroupId).toBe('shipsec-node-io-ingestor');
+    expect(service.kafkaClientId).toBe('shipsec-backend-node-io');
+  });
+
+  test('uses instance-scoped defaults when SHIPSEC_INSTANCE is set', () => {
+    process.env.SHIPSEC_INSTANCE = '4';
+    const repository = {
+      recordStart: async () => undefined,
+      recordCompletion: async () => undefined,
+    } as unknown as NodeIORepository;
+    const service = new NodeIOIngestService(repository) as unknown as {
+      kafkaGroupId: string;
+      kafkaClientId: string;
+    };
+
+    expect(service.kafkaGroupId).toBe('shipsec-node-io-ingestor-4');
+    expect(service.kafkaClientId).toBe('shipsec-backend-node-io-4');
+  });
+
+  test('prefers explicit env vars over defaults', () => {
+    process.env.SHIPSEC_INSTANCE = '9';
+    process.env.NODE_IO_KAFKA_GROUP_ID = 'custom-node-io-group';
+    process.env.NODE_IO_KAFKA_CLIENT_ID = 'custom-node-io-client';
+    const repository = {
+      recordStart: async () => undefined,
+      recordCompletion: async () => undefined,
+    } as unknown as NodeIORepository;
+    const service = new NodeIOIngestService(repository) as unknown as {
+      kafkaGroupId: string;
+      kafkaClientId: string;
+    };
+
+    expect(service.kafkaGroupId).toBe('custom-node-io-group');
+    expect(service.kafkaClientId).toBe('custom-node-io-client');
+  });
+});

backend/src/node-io/node-io-ingest.service.ts

@@ -46,9 +46,16 @@ export class NodeIOIngestService implements OnModuleInit, OnModuleDestroy {
     // Use instance-aware topic name
     const topicResolver = getTopicResolver();
     this.kafkaTopic = topicResolver.getNodeIOTopic();
+    const instanceId = process.env.SHIPSEC_INSTANCE;
+    const defaultGroupId = instanceId
+      ? `shipsec-node-io-ingestor-${instanceId}`
+      : 'shipsec-node-io-ingestor';
+    const defaultClientId = instanceId
+      ? `shipsec-backend-node-io-${instanceId}`
+      : 'shipsec-backend-node-io';
 
-    this.kafkaGroupId = process.env.NODE_IO_KAFKA_GROUP_ID ?? 'shipsec-node-io-ingestor';
-    this.kafkaClientId = process.env.NODE_IO_KAFKA_CLIENT_ID ?? 'shipsec-backend-node-io';
+    this.kafkaGroupId = process.env.NODE_IO_KAFKA_GROUP_ID ?? defaultGroupId;
+    this.kafkaClientId = process.env.NODE_IO_KAFKA_CLIENT_ID ?? defaultClientId;
   }
 
   async onModuleInit(): Promise<void> {

bun.lock

@@ -12,7 +12,7 @@ Security components wrap popular open-source tools for subdomain discovery, DNS
 ### Subfinder
 
 <Info>
-[GitHub](https://github.com/projectdiscovery/subfinder) · Docker: `projectdiscovery/subfinder`
+[GitHub](https://github.com/projectdiscovery/subfinder) · Docker: `ghcr.io/shipsecai/subfinder`
 </Info>
 
 Discovers subdomains using passive sources.
@@ -35,7 +35,7 @@ Discovers subdomains using passive sources.
 ### Amass
 
 <Info>
-[GitHub](https://github.com/owasp-amass/amass) · Docker: `owaspamass/amass`
+[GitHub](https://github.com/owasp-amass/amass) · Docker: `ghcr.io/shipsecai/amass`
 </Info>
 
 Active and passive subdomain enumeration.
@@ -73,7 +73,7 @@ High-performance DNS bruteforcing and resolution. This is a combined image that
 ### DNSX
 
 <Info>
-[GitHub](https://github.com/projectdiscovery/dnsx) · Docker: `projectdiscovery/dnsx`
+[GitHub](https://github.com/projectdiscovery/dnsx) · Docker: `ghcr.io/shipsecai/dnsx`
 </Info>
 
 Resolves DNS records with support for multiple record types and custom resolvers.
@@ -105,7 +105,7 @@ Resolves DNS records with support for multiple record types and custom resolvers
 ### httpx
 
 <Info>
-[GitHub](https://github.com/projectdiscovery/httpx) · Docker: `projectdiscovery/httpx`
+[GitHub](https://github.com/projectdiscovery/httpx) · Docker: `ghcr.io/shipsecai/httpx`
 </Info>
 
 Probes hosts for live HTTP services and captures response metadata.
@@ -137,7 +137,7 @@ Probes hosts for live HTTP services and captures response metadata.
 ### Naabu
 
 <Info>
-[GitHub](https://github.com/projectdiscovery/naabu) · Docker: `projectdiscovery/naabu`
+[GitHub](https://github.com/projectdiscovery/naabu) · Docker: `ghcr.io/shipsecai/naabu`
 </Info>
 
 Fast active port scanning using SYN/CONNECT probes.
@@ -196,7 +196,7 @@ Template-based vulnerability scanning. This is nuclei custom image with nuclei-t
 ### TruffleHog
 
 <Info>
-[GitHub](https://github.com/trufflesecurity/trufflehog) · Docker: `trufflesecurity/trufflehog`
+[GitHub](https://github.com/trufflesecurity/trufflehog) · Docker: `ghcr.io/shipsecai/trufflehog`
 </Info>
 
 Scans for leaked credentials across repositories, filesystems, and cloud storage.
@@ -227,7 +227,7 @@ Scans for leaked credentials across repositories, filesystems, and cloud storage
 ### Prowler Scan
 
 <Info>
-[GitHub](https://github.com/prowler-cloud/prowler) · Docker: `prowlercloud/prowler`
+[GitHub](https://github.com/prowler-cloud/prowler) · Docker: `ghcr.io/shipsecai/prowler`
 </Info>
 
 Cloud (AWS, Azure, GCP) security posture management. Best practices auditing.
@@ -262,7 +262,7 @@ Scans Supabase instances for misconfigurations.
 ### Notify
 
 <Info>
-[GitHub](https://github.com/projectdiscovery/notify) · Docker: `projectdiscovery/notify`
+[GitHub](https://github.com/projectdiscovery/notify) · Docker: `ghcr.io/shipsecai/notify`
 </Info>
 
 Sends alerts to Slack, Discord, Telegram, or email.

docs/components/security.mdx

@@ -892,7 +892,7 @@ export default defineComponent({
   category: 'security',
   runner: {
     kind: 'docker',
-    image: 'projectdiscovery/dnsx:latest',
+    image: 'ghcr.io/shipsecai/dnsx:latest',
     entrypoint: 'sh',
     command: ['-c', 'dnsx "$@"', '--'],
     network: 'bridge',

docs/development/component-development.mdx

@@ -13,6 +13,8 @@ import {
 import type { FrontendNodeData } from '@/schemas/node';
 import type { Node as ReactFlowNode, Edge as ReactFlowEdge } from 'reactflow';
 import { api } from '@/services/api';
+import { useSecretStore } from '@/store/secretStore';
+import { useComponentStore } from '@/store/componentStore';
 interface WorkflowMetadataShape {
   id: string | null;
   name: string;
@@ -131,6 +133,54 @@ export function useWorkflowImportExport({
         }),
       );
 
+      // Validate secret references
+      const removedSecrets: { param: string; node: string; secretId: string }[] = [];
+      try {
+        await useSecretStore.getState().fetchSecrets();
+        const secrets = useSecretStore.getState().secrets;
+        const secretIds = new Set(secrets.map((s) => s.id));
+
+        const componentStore = useComponentStore.getState();
+        if (Object.keys(componentStore.components).length === 0) {
+          await componentStore.fetchComponents();
+        }
+        const components = useComponentStore.getState().components;
+
+        resolvedNodes.forEach((node) => {
+          const data = node.data as FrontendNodeData;
+          const componentRef = data.componentId || data.componentSlug;
+          if (!componentRef) return;
+
+          const component =
+            componentStore.getComponent(componentRef) ||
+            Object.values(components).find((c) => c.slug === componentRef);
+
+          if (!component || !component.parameters) return;
+
+          // Find parameters that are secrets
+          const secretParams = component.parameters.filter((p) => p.type === 'secret');
+          const configParams = node.data.config.params || {};
+
+          secretParams.forEach((param) => {
+            const val = configParams[param.id];
+            // If value is a string (ID) and not in available secrets, remove it
+            if (typeof val === 'string' && val.trim().length > 0) {
+              if (!secretIds.has(val)) {
+                console.warn(
+                  `[Import] Removing invalid secret reference for param "${param.id}" in node "${node.id}" (secret ID: ${val})`,
+                );
+                removedSecrets.push({ param: param.id, node: node.id, secretId: val });
+                // Set to undefined to clear it
+                configParams[param.id] = undefined;
+              }
+            }
+          });
+        });
+      } catch (error) {
+        console.error('Failed to validate secrets during import:', error);
+        // Continue with import even if validation fails
+      }
+
       resetWorkflow();
       setDesignNodes(resolvedNodes as ReactFlowNode<FrontendNodeData>[]);
       setDesignEdges(normalizedEdges);
@@ -151,6 +201,15 @@ export function useWorkflowImportExport({
         title: 'Workflow imported',
         description: `Loaded ${parsed.name}`,
       });
+
+      // Show warning if any invalid secret references were removed
+      if (removedSecrets.length > 0) {
+        toast({
+          variant: 'warning',
+          title: 'Invalid secret references removed',
+          description: `${removedSecrets.length} secret reference(s) could not be resolved and were cleared. Please select valid secrets from the Secrets Manager.`,
+        });
+      }
     },
     [
       canManageWorkflows,