Fix/directory setting

dash_app/callbacks/callback_functions.py

@@ -6,6 +6,7 @@
 from dash_app.utils.data_directories import (
     get_all_filenames,
     get_all_root_log_directories,
+    get_base_path_directories,
     get_runs,
 )
 from dash_app.utils.metadata import format_metadata_rows
@@ -223,6 +224,11 @@ def get_log_data_directory_options(project_id: int):
     return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)]
 
 
+def get_base_path_options(selected_path=None):
+    labels, values = get_base_path_directories(selected_path)
+    return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)]
+
+
 def poll_task_status(task_id: int) -> dict:
     # There shouldn't be any error here unless the task_id is wrong.
     response, error = make_api_call({}, f"task-status/{task_id}", requests_type="GET")

dash_app/components/form_inputs.py

@@ -438,14 +438,24 @@ def base_path_input(id):
                 style={"textDecoration": "underline", "cursor": "pointer"},
             ),
             dbc.Tooltip(
-                "Optional. Must be directory inside log_data/. Leave empty or unchanged to use the default. Directory must already exist.",
+                "The directory the project's log data is read from. Every analysis in the "
+                "project picks its data from inside this directory, so keep the root to make "
+                "all of log_data/ available. The directories in log_data/ are listed here; "
+                "select one and open the list again to see the directories inside it.",
                 target=label_id,
                 placement="bottom",
             ),
-            dbc.Input(
-                id=id,
-                type="text",
-                value="./log_data",
+            dcc.Loading(
+                type="circle",
+                delay_show=200,
+                overlay_style={"visibility": "visible"},
+                children=dcc.Dropdown(
+                    id=id,
+                    placeholder="Select a directory under log_data/",
+                    className="dbc border border-light-subtle rounded",
+                    optionHeight=40,
+                    maxHeight=350,
+                ),
             ),
         ],
     )

dash_app/pages/home.py

@@ -2,7 +2,7 @@
 import dash_bootstrap_components as dbc
 from dash import ALL, Input, Output, State, callback, dcc
 
-from dash_app.callbacks.callback_functions import make_api_call
+from dash_app.callbacks.callback_functions import get_base_path_options, make_api_call
 from dash_app.components.forms import project_form
 from dash_app.components.layouts import create_home_layout
 from dash_app.utils.metadata import format_project_overview
@@ -59,6 +59,27 @@ def get_projects(_1, _2):
     return (group_items, dash.no_update, dash.no_update, dash.no_update, False)
 
 
+# Selecting a directory reveals the directories below it, so that deeper paths
+# can be reached by opening the dropdown again after selecting.
+@callback(
+    Output("base-path-proj", "options"),
+    Input("open-btn-proj", "n_clicks"),
+    Input("base-path-proj", "value"),
+)
+def get_base_paths(_, selected_path):
+    return get_base_path_options(selected_path)
+
+
+@callback(
+    Output("base-path-proj", "value"),
+    Input("base-path-proj", "id"),
+)
+def set_default_base_path(_):
+    # The log data root is the first option and the default base path
+    options = get_base_path_options()
+    return options[0]["value"] if options else None
+
+
 @callback(
     Output("collapse-proj", "is_open"),
     Input("open-btn-proj", "n_clicks"),

dash_app/utils/data_directories.py

@@ -49,3 +49,66 @@ def get_all_root_log_directories(base_path=None) -> tuple[list[str], list[str]]:
     paths = [os.path.abspath(base_path) + "/"] + directory_paths + file_paths
 
     return names, paths
+
+
+def get_base_path_directories(
+    selected_path: str | None = None, max_depth: int = 1
+) -> tuple[list[str], list[str]]:
+    """Directories that can be picked as a project base path.
+
+    Lists the log data root and the directories directly below it. When a
+    directory is already selected, the directories inside it are listed as
+    well, so that deeper paths are reached one level at a time by selecting a
+    directory and opening the dropdown again. Names are relative to the log
+    data root so that they read like the paths a user would type by hand.
+    """
+    root = os.path.abspath(current_app.config["LOG_DATA_PATH"])
+
+    if not os.path.isdir(root):
+        return [], []
+
+    paths = _walk_directories(root, max_depth)
+
+    if selected_path:
+        selected = os.path.abspath(selected_path)
+        if selected != root and _is_inside(selected, root):
+            # The selection and the directories leading to it stay listed, so
+            # that a deeper selection keeps showing in the dropdown and can be
+            # stepped back out of.
+            paths |= _path_and_parents(selected, root)
+            paths |= _walk_directories(selected, max_depth)
+
+    names = [f"{os.path.basename(root) or root} (root)"]
+    sorted_paths = [root] + sorted(paths)
+    names += [os.path.relpath(path, root) for path in sorted_paths[1:]]
+
+    return names, sorted_paths
+
+
+def _walk_directories(base_path: str, max_depth: int) -> set[str]:
+    directories = set()
+
+    for dirpath, dirnames, _ in os.walk(base_path):
+        depth = dirpath[len(base_path) :].count(os.sep)
+        if depth >= max_depth:
+            dirnames.clear()
+            continue
+
+        dirnames[:] = [dir for dir in dirnames if not dir.startswith(".")]
+        directories.update(os.path.join(dirpath, dir) for dir in dirnames)
+
+    return directories
+
+
+def _path_and_parents(path: str, root: str) -> set[str]:
+    paths = set()
+
+    while path != root and path != os.path.dirname(path):
+        paths.add(path)
+        path = os.path.dirname(path)
+
+    return paths
+
+
+def _is_inside(path: str, root: str) -> bool:
+    return os.path.commonpath([path, root]) == root

docker-compose-dev.yml

@@ -26,6 +26,8 @@ services:
     command: celery -A server.make_celery worker --concurrency=4 --loglevel=info
     volumes:
       - .:/app
+      - ${LOG_DATA_DIRECTORY}:/app/log_data/
+      - ${RESULTS_DIRECTORY}:/app/analysis_results
     environment:
       DB_URL: "${DB_URL}"
       CELERY_BROKER_URL: "${CELERY_BROKER_URL}"

docs/RunAgainstLogLeadBranch.md

@@ -0,0 +1,102 @@
+# Testing VisualLogAnalyzer against an unreleased LogLead branch
+
+Use this when you want to test LogLead changes on GitHub (a branch not yet
+released to PyPI) against VisualLogAnalyzer.
+
+## 1. Point requirements.txt at the branch
+
+Edit [`requirements.txt`](../requirements.txt) and replace the pinned line
+```
+LogLead==1.2.3
+```
+with a direct URL to a source tarball of the branch:
+
+```
+LogLead @ https://github.com/EvoTestOps/LogLead/archive/refs/heads/<BRANCH_NAME>.tar.gz
+```
+
+Swap `<BRANCH_NAME>` for whichever branch you're testing, e.g. `feature/more-loaders`.
+
+Note: this uses a plain HTTPS tarball, not a `git+https://...` reference —
+a `git+` URL would need the `git` binary inside the build image, which the
+`python:3.12-slim` base image doesn't include, and we don't want a
+permanent Dockerfile change just for occasional branch testing.
+
+## 2. Rebuild the Docker images
+
+LogLead is installed at image build time (`Dockerfile` runs
+`pip install -r requirements.txt`), so a container restart alone won't
+pick up the change — you need to rebuild:
+
+```bash
+docker compose -f docker-compose-dev.yml build app celery
+docker compose -f docker-compose-dev.yml up -d
+```
+
+(use `docker-compose.yml` instead of `docker-compose-dev.yml` if you're
+not using the dev override).
+
+If you re-run this later to pick up *new* commits pushed to the same
+branch, Docker's layer cache won't notice the branch moved (the
+requirements.txt text is unchanged), so force it:
+
+```bash
+docker compose -f docker-compose-dev.yml build --no-cache app celery
+```
+
+## 3. Verify what actually got installed
+
+```bash
+docker compose -f docker-compose-dev.yml exec app pip freeze | grep -i loglead
+```
+
+You should see the tarball URL you set in step 1, confirming the branch
+version installed rather than the PyPI release, e.g.:
+
+```
+LogLead @ https://github.com/EvoTestOps/LogLead/archive/refs/heads/feature/more-loaders.tar.gz
+```
+
+## 4. Automated smoke test
+
+Tests need the repo's bundled `log_data/LO2` mounted at `/app/log_data`, so (re)create the containers with `.env.sample` (`LOG_DATA_DIRECTORY=./log_data`) rather than your own `.env` — a plain `exec` on an already-running container won't change its mounts:
+
+```bash
+docker compose -f docker-compose-dev.yml --env-file .env.sample up -d --force-recreate app celery
+docker compose -f docker-compose-dev.yml exec app pytest tests
+curl -s http://localhost:5000/health
+```
+
+## 5. Switch back to your own data
+
+Recreate `app`/`celery` again, this time without `--env-file .env.sample`, so they pick up your regular `.env` (e.g. `LOG_DATA_DIRECTORY=~/Datasets`) instead:
+
+```bash
+docker compose -f docker-compose-dev.yml up -d --force-recreate app celery
+```
+
+## 6. Running and Manual Testing
+
+This repeats the relevant parts of [usage_guide.md](usage_guide.md) — see
+that file for the full walkthrough and screenshots.
+
+1. Make sure `LOG_DATA_DIRECTORY` and `RESULTS_DIRECTORY` in `.env` (or
+   `.env.sample`) point at the correct folders:
+   ```
+   LOG_DATA_DIRECTORY=./log_data
+   RESULTS_DIRECTORY=./analysis_results
+   ```
+2. Start the stack (already done in step 2 above if you rebuilt with
+   `up -d`; otherwise `docker compose -f docker-compose-dev.yml --env-file
+   .env up`).
+3. Open <http://localhost:5000/dash/> in a browser.
+
+## 7. Revert back to the released version when done
+
+```
+LogLead==1.2.3
+```
+
+then rebuild again (step 2). Don't leave a branch pin committed to `main`
+long-term — it's a floating target (CI and anyone else building the image
+would silently pick up whatever the branch currently contains).

tests/test_end_to_end.py

@@ -2,6 +2,19 @@
 from playwright.sync_api import Page, expect
 
 
+def select_base_path(page: Page, directory: str):
+    """Pick a directory in the project form's base path dropdown.
+
+    The options come from a callback and are virtualized, so wait for them to
+    arrive and type the directory to narrow the list down to the options that
+    are actually rendered.
+    """
+    page.click("#base-path-proj")
+    page.locator(".VirtualizedSelectOption").first.wait_for()
+    page.locator("#base-path-proj input").first.press_sequentially(directory)
+    page.locator(".VirtualizedSelectOption", has_text=directory).first.click()
+
+
 def test_has_title(page: Page):
     page.goto("http://127.0.0.1:5000/dash/")
 
@@ -24,7 +37,7 @@ def test_can_run_file_count_analysis(page: Page):
 
     page.get_by_role("button", name="Create a new project").click()
     page.get_by_label("Project name").fill("File count test")
-    page.get_by_label("Base path").fill("./log_data/LO2")
+    select_base_path(page, "LO2")
     page.get_by_role("button", name="Create", exact=True).click()
 
     project_link = page.locator("#project-group li a", has_text="File count test")
@@ -60,7 +73,7 @@ def test_can_run_ano_line_level(page: Page):
 
     page.get_by_role("button", name="Create a new project").click()
     page.get_by_label("Project name").fill("Ano Line Test")
-    page.get_by_label("Base path").fill("./log_data/LO2")
+    select_base_path(page, "LO2")
     page.get_by_role("button", name="Create", exact=True).click()
 
     project_link = page.locator("#project-group li a", has_text="Ano Line Test")
@@ -106,7 +119,7 @@ def test_error_ano_line_with_bad_inputs(page: Page):
 
     page.get_by_role("button", name="Create a new project").click()
     page.get_by_label("Project name").fill("Error Ano Line")
-    page.get_by_label("Base path").fill("./log_data/LO2")
+    select_base_path(page, "LO2")
     page.get_by_role("button", name="Create", exact=True).click()
 
     project_link = page.locator("#project-group li a", has_text="Error Ano Line")