Harden CLI/sdk e2e tests

Author: brandonkachen

cli/src/__tests__/e2e/cli-ui.test.ts

@@ -15,7 +15,9 @@ const TIMEOUT_MS = 25000
 const sdkBuilt = isSDKBuilt()
 
 if (!sdkBuilt) {
-  throw new Error('CLI UI tests require SDK to be built. Run: cd sdk && bun run build')
+  describe.skip('CLI UI Tests', () => {
+    test('skipped because SDK is not built', () => {})
+  })
 }
 
 let cliEnv: Record<string, string> = {}
@@ -99,6 +101,7 @@ describe('CLI UI Tests', () => {
 
           const text = await session.text()
           expect(text).toContain('--agent')
+          expect(text).toContain('--help')
         } finally {
           session.close()
         }
@@ -151,10 +154,10 @@ describe('CLI UI Tests', () => {
 
         try {
           // Commander should show an error for invalid flags
-          await session.waitForText('error', { timeout: 10000 })
+          await session.waitForText(/unknown option|error/i, { timeout: 10000 })
 
           const text = await session.text()
-          expect(text.toLowerCase()).toContain('error')
+          expect(text.toLowerCase()).toContain('unknown')
         } finally {
           session.close()
         }
@@ -170,12 +173,12 @@ describe('CLI UI Tests', () => {
         const session = await launchCLI({ args: [] })
 
         try {
-          // Wait for something to render - either the login modal or the main UI
-          // The CLI should render some UI within the timeout
-          await sleep(3000)
+          await session.waitForText(
+            /codebuff|login|directory|will run commands/i,
+            { timeout: 15000 },
+          )
 
           const text = await session.text()
-          // Should have rendered something - either login prompt or welcome message
           expect(text.length).toBeGreaterThan(0)
         } finally {
           await session.press(['ctrl', 'c'])
@@ -191,11 +194,10 @@ describe('CLI UI Tests', () => {
         const session = await launchCLI({ args: ['--agent', 'ask'] })
 
         try {
-          // CLI should start without errors
-          await sleep(2000)
+          await session.waitForText(/ask|codebuff|login/i, { timeout: 15000 })
 
           const text = await session.text()
-          expect(text.length).toBeGreaterThan(0)
+          expect(text.toLowerCase()).not.toContain('unknown option')
         } finally {
           await session.press(['ctrl', 'c'])
           session.close()
@@ -210,8 +212,9 @@ describe('CLI UI Tests', () => {
         const session = await launchCLI({ args: ['--clear-logs'] })
 
         try {
-          // CLI should start without errors
-          await sleep(2000)
+          await session.waitForText(/codebuff|login|directory/i, {
+            timeout: 15000,
+          })
 
           const text = await session.text()
           expect(text.length).toBeGreaterThan(0)
@@ -450,5 +453,3 @@ describe('CLI UI Tests', () => {
     )
   })
 })
-
-

cli/src/__tests__/e2e/full-stack.test.ts

@@ -36,12 +36,13 @@ function isDockerAvailable(): boolean {
 
 const dockerAvailable = isDockerAvailable()
 
-if (!sdkBuilt) {
-  throw new Error('E2E tests require SDK to be built. Run: cd sdk && bun run build')
-}
-
-if (!dockerAvailable) {
-  throw new Error('E2E tests require Docker to be running')
+if (!sdkBuilt || !dockerAvailable) {
+  const reason = !sdkBuilt
+    ? 'SDK not built (run: cd sdk && bun run build)'
+    : 'Docker not running'
+  describe.skip(`E2E skipped: ${reason}`, () => {
+    test('skipped', () => {})
+  })
 }
 
 describe('E2E: Chat Interaction', () => {
@@ -64,13 +65,13 @@ describe('E2E: Chat Interaction', () => {
     async () => {
       const session = await ctx.createSession()
 
-      // Wait for CLI to render
-      await sleep(5000)
-
+      await session.cli.waitForText(/codebuff|login|directory|will run/i, {
+        timeout: 15000,
+      })
       const text = await session.cli.text()
-      // Should show Codebuff branding or welcome message
       const hasWelcome =
         text.toLowerCase().includes('codebuff') ||
+        text.toLowerCase().includes('login') ||
         text.includes('Directory') ||
         text.includes('will run commands')
       expect(hasWelcome).toBe(true)
@@ -83,14 +84,11 @@ describe('E2E: Chat Interaction', () => {
     async () => {
       const session = await ctx.createSession()
 
-      await sleep(5000)
-
       // Type a test message
       await session.cli.type('Hello from e2e test')
-      await sleep(500)
-
-      const text = await session.cli.text()
-      expect(text).toContain('Hello from e2e test')
+      await session.cli.waitForText('Hello from e2e test', {
+        timeout: 10000,
+      })
     },
     TIMEOUT_MS,
   )
@@ -100,24 +98,14 @@ describe('E2E: Chat Interaction', () => {
     async () => {
       const session = await ctx.createSession()
 
-      await sleep(5000)
-
       // Type and send a message
       await session.cli.type('What is 2+2?')
       await sleep(300)
       await session.cli.press('enter')
 
-      // Wait for status to appear
-      await sleep(2000)
-
-      const text = await session.cli.text()
-      // Should show some status indicator
-      const hasStatus =
-        text.includes('thinking') ||
-        text.includes('working') ||
-        text.includes('connecting') ||
-        text.includes('What is 2+2?') // Message should at least appear
-      expect(hasStatus).toBe(true)
+      await session.cli.waitForText(/thinking|working|connecting|2\+2/i, {
+        timeout: 15000,
+      })
     },
     TIMEOUT_MS,
   )
@@ -143,17 +131,13 @@ describe('E2E: Slash Commands', () => {
     async () => {
       const session = await ctx.createSession()
 
-      await sleep(5000)
-
       // Type /new and press enter
       await session.cli.type('/new')
       await sleep(300)
       await session.cli.press('enter')
-      await sleep(1500)
-
-      const text = await session.cli.text()
-      // CLI should still be running
-      expect(text.length).toBeGreaterThan(0)
+      await session.cli.waitForText(/\/new|conversation/i, {
+        timeout: 10000,
+      })
     },
     TIMEOUT_MS,
   )
@@ -163,22 +147,11 @@ describe('E2E: Slash Commands', () => {
     async () => {
       const session = await ctx.createSession()
 
-      await sleep(5000)
-
       // Type /usage and press enter
       await session.cli.type('/usage')
       await sleep(300)
       await session.cli.press('enter')
-      await sleep(2000)
-
-      const text = await session.cli.text()
-      // Should show some credit-related information
-      const hasUsageInfo =
-        text.toLowerCase().includes('credit') ||
-        text.toLowerCase().includes('usage') ||
-        text.includes('1000') || // Test user has 1000 credits
-        text.includes('/usage')
-      expect(hasUsageInfo).toBe(true)
+      await session.cli.waitForText(/credit|usage|1000/i, { timeout: 15000 })
     },
     TIMEOUT_MS,
   )
@@ -188,8 +161,6 @@ describe('E2E: Slash Commands', () => {
     async () => {
       const session = await ctx.createSession()
 
-      await sleep(5000)
-
       // Type / to trigger suggestions
       await session.cli.type('/')
       await sleep(1000)
@@ -884,5 +855,3 @@ describe('E2E: Error Scenarios', () => {
     TIMEOUT_MS,
   )
 })
-
-

sdk/e2e/features/knowledge-files.e2e.test.ts

@@ -18,18 +18,27 @@ import {
 
 describe('Features: Knowledge Files', () => {
   let client: CodebuffClient
+  let apiKey: string | null = null
 
   beforeAll(() => {
+    apiKey = process.env.CODEBUFF_API_KEY ?? null
+    if (!apiKey) {
+      // Skip gracefully if no API key is configured
+      test.skip('CODEBUFF_API_KEY is required for knowledge files e2e')
+      return
+    }
     client = new CodebuffClient({ apiKey: getApiKey() })
   })
 
   beforeEach(async () => {
+    if (!apiKey) return
     await ensureBackendConnection()
   })
 
   test(
     'agent uses injected knowledge files',
     async () => {
+      if (!apiKey) return
       const collector = new EventCollector()
 
       const result = await client.run({
@@ -43,15 +52,20 @@ describe('Features: Knowledge Files', () => {
 
       if (isAuthError(result.output)) return
 
-      if (result.output.type === 'error') return
-      expect(collector.hasEventType('finish')).toBe(true)
+      expect(result.output.type).not.toBe('error')
+      const responseText = collector.getFullText().toUpperCase()
+      expect(
+        responseText.includes('PINEAPPLE42') ||
+          responseText.includes('PINEAPPLE'),
+      ).toBe(true)
     },
     DEFAULT_TIMEOUT,
   )
 
   test(
     'multiple knowledge files are accessible',
     async () => {
+      if (!apiKey) return
       const collector = new EventCollector()
 
       const result = await client.run({
@@ -68,8 +82,12 @@ describe('Features: Knowledge Files', () => {
 
       if (isAuthError(result.output)) return
 
-      if (result.output.type === 'error') return
-      expect(collector.hasEventType('finish')).toBe(true)
+      expect(result.output.type).not.toBe('error')
+      const responseText = collector.getFullText().toLowerCase()
+      expect(
+        responseText.includes('innovation') ||
+          responseText.includes('integrity'),
+      ).toBe(true)
     },
     DEFAULT_TIMEOUT,
   )

sdk/e2e/features/project-files.e2e.test.ts

@@ -19,18 +19,26 @@ import {
 
 describe('Features: Project Files', () => {
   let client: CodebuffClient
+  let apiKey: string | null = null
 
   beforeAll(() => {
+    apiKey = process.env.CODEBUFF_API_KEY ?? null
+    if (!apiKey) {
+      test.skip('CODEBUFF_API_KEY is required for project files e2e')
+      return
+    }
     client = new CodebuffClient({ apiKey: getApiKey() })
   })
 
   beforeEach(async () => {
+    if (!apiKey) return
     await ensureBackendConnection()
   })
 
   test(
     'agent can reference injected project files',
     async () => {
+      if (!apiKey) return
       const collector = new EventCollector()
 
       const result = await client.run({
@@ -42,15 +50,22 @@ describe('Features: Project Files', () => {
 
       if (isAuthError(result.output)) return
 
-      if (result.output.type === 'error') return
-      expect(collector.hasEventType('finish')).toBe(true)
+      expect(result.output.type).not.toBe('error')
+      const responseText = collector.getFullText().toLowerCase()
+      expect(
+        responseText.includes('index') ||
+          responseText.includes('calculator') ||
+          responseText.includes('package.json') ||
+          responseText.includes('readme'),
+      ).toBe(true)
     },
     DEFAULT_TIMEOUT,
   )
 
   test(
     'agent can analyze content of project files',
     async () => {
+      if (!apiKey) return
       const collector = new EventCollector()
 
       const result = await client.run({
@@ -63,7 +78,12 @@ describe('Features: Project Files', () => {
       if (isAuthError(result.output)) return
 
       expect(result.output.type).not.toBe('error')
-      expect(collector.hasEventType('finish')).toBe(true)
+      const responseText = collector.getFullText().toLowerCase()
+      expect(
+        responseText.includes('calculator') ||
+          responseText.includes('add') ||
+          responseText.includes('result'),
+      ).toBe(true)
     },
     DEFAULT_TIMEOUT,
   )