File system cleanup

Author: Zia-Rashid

Sources/Agentic_System/.gitignore

@@ -8,7 +8,7 @@
 3. Put your OpenAI key into a `keys.cfg` in Sources/Agentic_System
 4. Replace the smolagents site-packaged located in `.venv/lib/python3.12/site-packages` or similar with the provided fork of smolagents<br>
     You can simply remove the existing smolagents in site-packages and move + rename the fork as `smolagents`
-5. run `python3 rises-the-fog.py (--debug)`
+5. run `python3 start_scripts/rises-the-fog.py (--debug)`
 
 ### Technical flow
 #### The first multi agent system is implemented and starts by initializing a root manager whose goal is to actually orchestrate the creation of program templates. It starts by selecting a "code region" that it determines to be interesting; this is done by querying a RAG DB (json file) that contains over 8000 regression tests, their FuzzIL form, and execution data via trace flags.  We instruct the system to select a code region by using the execution data. On top of that, the system has access to a vector RAG DB with: V8 docs, JS MDM docs, C++ docs, and various research papers that it can query to gather more information. The vectorization library we use is META’s FAISS -"Facebook AI Similarity Search". After this is done it will select a code region such as: "Keyed array element access & elements-kind transitions (KeyedStoreIC/KeyedLoadIC, ElementsTransition, GrowElements/CopyElements, and Array builtin fast paths)".

Sources/Agentic_System/README.md

@@ -3,7 +3,7 @@
 from pathlib import Path
 
 _module_dir = Path(__file__).parent
-_module_path = _module_dir / "rises-the-fog.py"
+_module_path = _module_dir / "start_scripts" / "rises-the-fog.py"
 
 if str(_module_dir) not in sys.path:
     sys.path.insert(0, str(_module_dir))

Sources/Agentic_System/__init__.py

@@ -6,14 +6,23 @@
 import sys
 import subprocess
 from pathlib import Path
-import logging 
+import logging
 from datetime import datetime
 import pytz
-import config_loader as config_loader 
+
+import site
+
+# Ensure the Agentic_System package root is on sys.path so sibling modules import correctly
+_agentic_root = Path(__file__).resolve().parents[1]
+if str(_agentic_root) not in sys.path:
+    sys.path.insert(0, str(_agentic_root))
+
+import config_loader as config_loader
 from agents.EBG_crash import EBG_Crash
 from agents.EBG_plateau import EBG_Plateau
 from smolagents import LiteLLMModel
 from config_loader import get_openai_api_key, get_anthropic_api_key, get_deepseek_api_key
+
 logger = logging.getLogger("boiled_eggs")
 if not logger.handlers:
     logger.addHandler(logging.NullHandler())
@@ -23,12 +32,10 @@
 
 BASE_MODEL_ID = "deepseek"
 
-import site
-from pathlib import Path
-
 # Prefer the project's virtualenv site-packages if present, so tools like chromadb are importable
 try:
-    _root = Path(__file__).resolve().parents[2]
+    # Repo root (unchanged despite this file moving one level deeper)
+    _root = Path(__file__).resolve().parents[3]
     _venv_site = _root / ".venv" / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages"
     if _venv_site.exists():
         site.addsitedir(str(_venv_site))
@@ -42,30 +49,33 @@ def __init__(self, mode: str = "Crash", fuzzer_id: str = "fuzzer-1", crash_progr
         self.openai_api_key = get_openai_api_key()
         self.anthropic_api_key = get_anthropic_api_key()
         self.deepseek_api_key = get_deepseek_api_key()
-        
+
         if self.deepseek_api_key:
             os.environ["DEEPSEEK_API_KEY"] = self.deepseek_api_key
-        
+
         self.model = LiteLLMModel(model_id=BASE_MODEL_ID, api_key=self.openai_api_key)
         print("System is running in " + mode + " mode")
         if mode == "Crash":
             self.system = EBG_Crash(self.model, api_key=self.openai_api_key, anthropic_api_key=self.anthropic_api_key, crash_program_hash=crash_program_hash)
         elif mode == "Plateau":
             self.system = EBG_Plateau(self.model, api_key=self.openai_api_key, anthropic_api_key=self.anthropic_api_key, fuzzer_id=fuzzer_id)
 
-def run(force_logging: bool = True):
 
-    site.addsitedir(Path(__file__).parent.parent)
-    #smolagent-fork
+def run(force_logging: bool = True):
+    # Add the previous parent directory (Sources) to site dirs, preserving old behavior
+    site.addsitedir(Path(__file__).resolve().parents[2])
+    # smolagent-fork
 
     parser = argparse.ArgumentParser(description="Ethiopian Boiled Eggs agentic system")
     parser.add_argument("--debug", action="store_true", help="Enable debug logging to fog logs")
     args = parser.parse_args()
-    #force logging 
+    # force logging
     args.debug = force_logging
 
     if args.debug:
-        log_dir = Path(__file__).parent / 'agents' / 'ebg_logs'
+        # Logs live under Agentic_System/agents/ebg_logs even though this file moved into start_scripts
+        agentic_root = Path(__file__).resolve().parents[1]
+        log_dir = agentic_root / 'agents' / 'ebg_logs'
         log_dir.mkdir(parents=True, exist_ok=True)
         latest_num = 0
         if os.path.exists(log_dir / 'ethiopian_boiled_egg.log'):
@@ -77,13 +87,12 @@ def run(force_logging: bool = True):
                             if num > latest_num:
                                 latest_num = num
             log_path = str(log_dir / f'ethiopian_boiled_egg{latest_num + 1}.log')
-        else: 
+        else:
             log_path = str(log_dir / f'ethiopian_boiled_egg.log')
 
         if os.path.exists(log_path):
             print(f"Log file already exists: {log_path}")
 
-                        
         # Configure logger to write messages as-is (no prefixes) for 1:1 capture
         logger.handlers.clear()
         file_handler = logging.FileHandler(log_path, mode='a', encoding='utf-8')
@@ -92,22 +101,24 @@ def run(force_logging: bool = True):
         logger.setLevel(logging.INFO)
         logger.disabled = False
 
-        
         class _StreamToLogger:
             def __init__(self, log_fn):
                 self.log_fn = log_fn
                 self._buffer = ''
+
             def write(self, message):
                 if not isinstance(message, str):
                     message = message.decode('utf-8', errors='ignore')
                 self._buffer += message
                 while '\n' in self._buffer:
                     line, self._buffer = self._buffer.split('\n', 1)
                     self.log_fn(line)
+
             def flush(self):
                 if self._buffer:
                     self.log_fn(self._buffer)
                     self._buffer = ''
+
             def isatty(self):
                 return False
 
@@ -121,16 +132,19 @@ def isatty(self):
     logger.info(f"time: {datetime.now(est_timezone)}")
     a = EthiopianBoiledEgg(mode="Plateau")
     path = os.path.join(os.getenv('FUZZILLI_PATH', ''), "Sources", "Agentic_System")
-    if (not os.path.exists(os.path.join(path, "regressions.json"))):
+    regressions_dir = os.path.join(path, "regressions")
+    if not os.path.exists(os.path.join(regressions_dir, "regressions.json")):
         try:
-            subprocess.run(["unzstd", os.path.join(path, "regressions.json.zst")], check=True)
-            #unzstd regressions.json.zst
+            subprocess.run(["unzstd", os.path.join(regressions_dir, "regressions.json.zst")], check=True)
+            # unzstd regressions.json.zst
         except subprocess.CalledProcessError as e:
             logger.error(f"Error decompressing regressions.json.zst: {e}")
             exit(1)
         else:
             logger.info("Regressions.json decompressed successfully")
     a.system.start_system()
 
+
 if __name__ == "__main__":
-    sys.exit(run())
+    sys.exit(run())
+

…rces/Agentic_System/regressions.json.zst …_System/regressions/regressions.json.zstSources/Agentic_System/regressions.json.zst renamed to Sources/Agentic_System/regressions/regressions.json.zst Sources/Agentic_System/regressions.json.zst renamed to Sources/Agentic_System/regressions/regressions.json.zst

@@ -6,14 +6,23 @@
 import sys
 import subprocess
 from pathlib import Path
-import logging 
+import logging
 from datetime import datetime
 import pytz
-import config_loader as config_loader 
+
+import site
+
+# Ensure the Agentic_System package root is on sys.path so sibling modules import correctly
+_agentic_root = Path(__file__).resolve().parents[1]
+if str(_agentic_root) not in sys.path:
+    sys.path.insert(0, str(_agentic_root))
+
+import config_loader as config_loader
 from agents.FoG import Father
-#from agents.EBG import EBG
+# from agents.EBG import EBG
 from smolagents import LiteLLMModel
 from config_loader import get_openai_api_key, get_anthropic_api_key, get_deepseek_api_key
+
 logger = logging.getLogger("rises_the_fog")
 if not logger.handlers:
     logger.addHandler(logging.NullHandler())
@@ -23,12 +32,10 @@
 
 BASE_MODEL_ID = "gpt-5-mini"
 
-import site
-from pathlib import Path
-
 # Prefer the project's virtualenv site-packages if present, so tools like chromadb are importable
 try:
-    _root = Path(__file__).resolve().parents[2]
+    # Repo root (unchanged despite this file moving one level deeper)
+    _root = Path(__file__).resolve().parents[3]
     _venv_site = _root / ".venv" / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages"
     if _venv_site.exists():
         site.addsitedir(str(_venv_site))
@@ -42,28 +49,30 @@ def __init__(self):
         self.openai_api_key = get_openai_api_key()
         self.anthropic_api_key = get_anthropic_api_key()
         self.deepseek_api_key = get_deepseek_api_key()
-        
+
         if self.deepseek_api_key:
             os.environ["DEEPSEEK_API_KEY"] = self.deepseek_api_key
-        
+
         self.model = LiteLLMModel(model_id=BASE_MODEL_ID, api_key=self.openai_api_key)
         self.system = Father(self.model, api_key=self.deepseek_api_key, anthropic_api_key=self.anthropic_api_key)
         # self.ebg = EBG(self.model, api_key=self.openai_api_key, anthropic_api_key=self.anthropic_api_key)
-        
 
-def run(force_logging: bool = True):
 
-    site.addsitedir(Path(__file__).parent.parent)
-    #smolagent-fork
+def run(force_logging: bool = True):
+    # Add the previous parent directory (Sources) to site dirs, preserving old behavior
+    site.addsitedir(Path(__file__).resolve().parents[2])
+    # smolagent-fork
 
     parser = argparse.ArgumentParser(description="Rise the FoG agentic system")
     parser.add_argument("--debug", action="store_true", help="Enable debug logging to fog logs")
     args = parser.parse_args()
-    #force logging 
+    # force logging
     args.debug = force_logging
 
     if args.debug:
-        log_dir = Path(__file__).parent / 'agents' / 'fog_logs'
+        # Logs live under Agentic_System/agents/fog_logs even though this file moved into start_scripts
+        agentic_root = Path(__file__).resolve().parents[1]
+        log_dir = agentic_root / 'agents' / 'fog_logs'
         log_dir.mkdir(parents=True, exist_ok=True)
         latest_num = 0
         if os.path.exists(log_dir / 'rises_the_fog.log'):
@@ -75,13 +84,12 @@ def run(force_logging: bool = True):
                             if num > latest_num:
                                 latest_num = num
             log_path = str(log_dir / f'rises_the_fog{latest_num + 1}.log')
-        else: 
+        else:
             log_path = str(log_dir / f'rises_the_fog.log')
 
         if os.path.exists(log_path):
             print(f"Log file already exists: {log_path}")
 
-                        
         # Configure logger to write messages as-is (no prefixes) for 1:1 capture
         logger.handlers.clear()
         file_handler = logging.FileHandler(log_path, mode='a', encoding='utf-8')
@@ -90,22 +98,24 @@ def run(force_logging: bool = True):
         logger.setLevel(logging.INFO)
         logger.disabled = False
 
-        
         class _StreamToLogger:
             def __init__(self, log_fn):
                 self.log_fn = log_fn
                 self._buffer = ''
+
             def write(self, message):
                 if not isinstance(message, str):
                     message = message.decode('utf-8', errors='ignore')
                 self._buffer += message
                 while '\n' in self._buffer:
                     line, self._buffer = self._buffer.split('\n', 1)
                     self.log_fn(line)
+
             def flush(self):
                 if self._buffer:
                     self.log_fn(self._buffer)
                     self._buffer = ''
+
             def isatty(self):
                 return False
 
@@ -119,10 +129,11 @@ def isatty(self):
     logger.info(f"time: {datetime.now(est_timezone)}")
     a = FatherOfGod()
     path = os.path.join(os.getenv('FUZZILLI_PATH', ''), "Sources", "Agentic_System")
-    if (not os.path.exists(os.path.join(path, "regressions.json"))):
+    regressions_dir = os.path.join(path, "regressions")
+    if not os.path.exists(os.path.join(regressions_dir, "regressions.json")):
         try:
-            subprocess.run(["unzstd", os.path.join(path, "regressions.json.zst")], check=True)
-            #unzstd regressions.json.zst
+            subprocess.run(["unzstd", os.path.join(regressions_dir, "regressions.json.zst")], check=True)
+            # unzstd regressions.json.zst
         except subprocess.CalledProcessError as e:
             logger.error(f"Error decompressing regressions.json.zst: {e}")
             exit(1)
@@ -133,3 +144,4 @@ def isatty(self):
 
 if __name__ == "__main__":
     sys.exit(run())
+

…ces/Agentic_System/regressions2.json.zst …System/regressions/regressions2.json.zstSources/Agentic_System/regressions2.json.zst renamed to Sources/Agentic_System/regressions/regressions2.json.zst Sources/Agentic_System/regressions2.json.zst renamed to Sources/Agentic_System/regressions/regressions2.json.zst

@@ -23,7 +23,7 @@
     pass
 
 # Cached regressions.json data to avoid reloading on every tool call
-_REGRESSIONS_PATH = (Path(__file__).parent.parent / "regressions.json").resolve()
+_REGRESSIONS_PATH = (Path(__file__).parent.parent / "regressions" / "regressions.json").resolve()
 _REGRESSIONS_CACHE = None
 _TEMPLATES_PATH = (Path(__file__).parent.parent / "templates" / "templates.json").resolve()
 _TEMPLATES_CACHE = None