Merge pull request #263 from tcdent/test-frameworks-dependencies

Create separate test environments for each framework.

Author: bboynton97

.github/workflows/python-testing.yml

@@ -25,15 +25,17 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.10,3.11,3.12]
+        python-version: ["3.10","3.11","3.12"]  # versions need to be strings
 
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.11' # Use a default Python version for running tox
+          python-version: ${{ matrix.python-version }}
       - name: Install tox
         run: pip install tox
+      - name: Install tox-gh-actions
+        run: pip install tox-gh-actions  # allows tox to get the current python version
       - name: Run tests with tox
-        run: tox
+        run: tox  # we can't pass arguments to tox when using tox-gh-actions, so all frameworks happen in one run

pyproject.toml

@@ -46,14 +46,12 @@ test = [
 crewai = [
     "crewai==0.100.0",
     "crewai-tools==0.33.0",
+    "shapely==2.0.6"  # crewai-tools needs this, but 2.0.7 is broken
 ]
 langgraph = [
     "langgraph>=0.2.61",
     "langchain>=0.3.14",
 ]
-all = [
-    "agentstack[dev,test,crewai,langgraph]",
-]
 
 [tool.setuptools.package-data]
 agentstack = ["templates/**/*"]

tests/test_cli_templates.py

@@ -11,7 +11,8 @@
 
 class CLITemplatesTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = Path(BASE_PATH / 'tmp/cli_templates')
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'cli_templates'
         os.makedirs(self.project_dir, exist_ok=True)
         os.chdir(self.project_dir)
 

tests/test_cli_tools.py

@@ -17,7 +17,8 @@
 # TODO parameterized framework
 class CLIToolsTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = Path(BASE_PATH / 'tmp/cli_tools')
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'cli_tools'
         os.makedirs(self.project_dir, exist_ok=True)
         os.chdir(self.project_dir)
 

tests/test_frameworks.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized, parameterized_class
+from parameterized import parameterized
 
 from agentstack.conf import ConfigFile, set_path
 from agentstack.exceptions import ValidationError
@@ -16,9 +16,9 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestFrameworks(unittest.TestCase):
     def setUp(self):
+        self.framework = os.getenv('TEST_FRAMEWORK')
         self.project_dir = BASE_PATH / 'tmp' / self.framework / 'test_frameworks'
 
         os.makedirs(self.project_dir)

tests/test_frameworks_langgraph.py

@@ -18,7 +18,8 @@
 
 class FrameworksLanggraphTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp/frameworks/langgraph'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'langgraph'
         conf.set_path(self.project_dir)
         os.makedirs(self.project_dir / 'src/config')
 

tests/test_generation_agent.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized_class
 import ast
 
 from agentstack.conf import ConfigFile, set_path
@@ -15,10 +14,10 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestGenerationAgent(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp' / 'agent_generation'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'agent_generation'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')

tests/test_generation_files.py

@@ -18,7 +18,8 @@
 # TODO copy files to working directory
 class GenerationFilesTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / "tmp" / "generation_files"
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'generation_files'
         os.makedirs(self.project_dir)
 
         shutil.copy(BASE_PATH / "fixtures/agentstack.json", self.project_dir / "agentstack.json")

tests/test_generation_tasks.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized_class
 import ast
 
 from agentstack.conf import ConfigFile, set_path
@@ -16,10 +15,10 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestGenerationAgent(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp' / 'agent_generation'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'agent_generation'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')

tests/test_generation_tool.py

@@ -15,10 +15,10 @@
 
 
 # TODO parameterize all tools
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestGenerationTool(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp' / 'tool_generation'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'tool_generation'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')