From 4dc753685a79fc7598783942c705f70132288596 Mon Sep 17 00:00:00 2001 From: mmantyla Date: Thu, 20 Aug 2026 09:46:57 +0300 Subject: [PATCH 1/2] How to run agains LogLead branch and dev data mount fix --- docker-compose-dev.yml | 2 + docs/RunAgainstLogLeadBranch.md | 102 ++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 docs/RunAgainstLogLeadBranch.md diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 67120d5..2237235 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -26,6 +26,8 @@ services: command: celery -A server.make_celery worker --concurrency=4 --loglevel=info volumes: - .:/app + - ${LOG_DATA_DIRECTORY}:/app/log_data/ + - ${RESULTS_DIRECTORY}:/app/analysis_results environment: DB_URL: "${DB_URL}" CELERY_BROKER_URL: "${CELERY_BROKER_URL}" diff --git a/docs/RunAgainstLogLeadBranch.md b/docs/RunAgainstLogLeadBranch.md new file mode 100644 index 0000000..43ca9fe --- /dev/null +++ b/docs/RunAgainstLogLeadBranch.md @@ -0,0 +1,102 @@ +# Testing VisualLogAnalyzer against an unreleased LogLead branch + +Use this when you want to test LogLead changes on GitHub (a branch not yet +released to PyPI) against VisualLogAnalyzer. + +## 1. Point requirements.txt at the branch + +Edit [`requirements.txt`](../requirements.txt) and replace the pinned line +``` +LogLead==1.2.3 +``` +with a direct URL to a source tarball of the branch: + +``` +LogLead @ https://github.com/EvoTestOps/LogLead/archive/refs/heads/.tar.gz +``` + +Swap `` for whichever branch you're testing, e.g. `feature/more-loaders`. + +Note: this uses a plain HTTPS tarball, not a `git+https://...` reference — +a `git+` URL would need the `git` binary inside the build image, which the +`python:3.12-slim` base image doesn't include, and we don't want a +permanent Dockerfile change just for occasional branch testing. + +## 2. Rebuild the Docker images + +LogLead is installed at image build time (`Dockerfile` runs +`pip install -r requirements.txt`), so a container restart alone won't +pick up the change — you need to rebuild: + +```bash +docker compose -f docker-compose-dev.yml build app celery +docker compose -f docker-compose-dev.yml up -d +``` + +(use `docker-compose.yml` instead of `docker-compose-dev.yml` if you're +not using the dev override). + +If you re-run this later to pick up *new* commits pushed to the same +branch, Docker's layer cache won't notice the branch moved (the +requirements.txt text is unchanged), so force it: + +```bash +docker compose -f docker-compose-dev.yml build --no-cache app celery +``` + +## 3. Verify what actually got installed + +```bash +docker compose -f docker-compose-dev.yml exec app pip freeze | grep -i loglead +``` + +You should see the tarball URL you set in step 1, confirming the branch +version installed rather than the PyPI release, e.g.: + +``` +LogLead @ https://github.com/EvoTestOps/LogLead/archive/refs/heads/feature/more-loaders.tar.gz +``` + +## 4. Automated smoke test + +Tests need the repo's bundled `log_data/LO2` mounted at `/app/log_data`, so (re)create the containers with `.env.sample` (`LOG_DATA_DIRECTORY=./log_data`) rather than your own `.env` — a plain `exec` on an already-running container won't change its mounts: + +```bash +docker compose -f docker-compose-dev.yml --env-file .env.sample up -d --force-recreate app celery +docker compose -f docker-compose-dev.yml exec app pytest tests +curl -s http://localhost:5000/health +``` + +## 5. Switch back to your own data + +Recreate `app`/`celery` again, this time without `--env-file .env.sample`, so they pick up your regular `.env` (e.g. `LOG_DATA_DIRECTORY=~/Datasets`) instead: + +```bash +docker compose -f docker-compose-dev.yml up -d --force-recreate app celery +``` + +## 6. Running and Manual Testing + +This repeats the relevant parts of [usage_guide.md](usage_guide.md) — see +that file for the full walkthrough and screenshots. + +1. Make sure `LOG_DATA_DIRECTORY` and `RESULTS_DIRECTORY` in `.env` (or + `.env.sample`) point at the correct folders: + ``` + LOG_DATA_DIRECTORY=./log_data + RESULTS_DIRECTORY=./analysis_results + ``` +2. Start the stack (already done in step 2 above if you rebuilt with + `up -d`; otherwise `docker compose -f docker-compose-dev.yml --env-file + .env up`). +3. Open in a browser. + +## 7. Revert back to the released version when done + +``` +LogLead==1.2.3 +``` + +then rebuild again (step 2). Don't leave a branch pin committed to `main` +long-term — it's a floating target (CI and anyone else building the image +would silently pick up whatever the branch currently contains). From c2c89a7360446ee3ec3f9611ea408f9e455038f0 Mon Sep 17 00:00:00 2001 From: mmantyla Date: Thu, 20 Aug 2026 10:56:30 +0300 Subject: [PATCH 2/2] Directory browsing in project creation screen --- dash_app/callbacks/callback_functions.py | 6 +++ dash_app/components/form_inputs.py | 20 ++++++-- dash_app/pages/home.py | 23 ++++++++- dash_app/utils/data_directories.py | 63 ++++++++++++++++++++++++ tests/test_end_to_end.py | 19 +++++-- 5 files changed, 122 insertions(+), 9 deletions(-) diff --git a/dash_app/callbacks/callback_functions.py b/dash_app/callbacks/callback_functions.py index 763449a..3da32c5 100644 --- a/dash_app/callbacks/callback_functions.py +++ b/dash_app/callbacks/callback_functions.py @@ -6,6 +6,7 @@ from dash_app.utils.data_directories import ( get_all_filenames, get_all_root_log_directories, + get_base_path_directories, get_runs, ) from dash_app.utils.metadata import format_metadata_rows @@ -223,6 +224,11 @@ def get_log_data_directory_options(project_id: int): return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)] +def get_base_path_options(selected_path=None): + labels, values = get_base_path_directories(selected_path) + return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)] + + def poll_task_status(task_id: int) -> dict: # There shouldn't be any error here unless the task_id is wrong. response, error = make_api_call({}, f"task-status/{task_id}", requests_type="GET") diff --git a/dash_app/components/form_inputs.py b/dash_app/components/form_inputs.py index 770be80..8244f57 100644 --- a/dash_app/components/form_inputs.py +++ b/dash_app/components/form_inputs.py @@ -438,14 +438,24 @@ def base_path_input(id): style={"textDecoration": "underline", "cursor": "pointer"}, ), dbc.Tooltip( - "Optional. Must be directory inside log_data/. Leave empty or unchanged to use the default. Directory must already exist.", + "The directory the project's log data is read from. Every analysis in the " + "project picks its data from inside this directory, so keep the root to make " + "all of log_data/ available. The directories in log_data/ are listed here; " + "select one and open the list again to see the directories inside it.", target=label_id, placement="bottom", ), - dbc.Input( - id=id, - type="text", - value="./log_data", + dcc.Loading( + type="circle", + delay_show=200, + overlay_style={"visibility": "visible"}, + children=dcc.Dropdown( + id=id, + placeholder="Select a directory under log_data/", + className="dbc border border-light-subtle rounded", + optionHeight=40, + maxHeight=350, + ), ), ], ) diff --git a/dash_app/pages/home.py b/dash_app/pages/home.py index 71b711a..9ac050a 100644 --- a/dash_app/pages/home.py +++ b/dash_app/pages/home.py @@ -2,7 +2,7 @@ import dash_bootstrap_components as dbc from dash import ALL, Input, Output, State, callback, dcc -from dash_app.callbacks.callback_functions import make_api_call +from dash_app.callbacks.callback_functions import get_base_path_options, make_api_call from dash_app.components.forms import project_form from dash_app.components.layouts import create_home_layout from dash_app.utils.metadata import format_project_overview @@ -59,6 +59,27 @@ def get_projects(_1, _2): return (group_items, dash.no_update, dash.no_update, dash.no_update, False) +# Selecting a directory reveals the directories below it, so that deeper paths +# can be reached by opening the dropdown again after selecting. +@callback( + Output("base-path-proj", "options"), + Input("open-btn-proj", "n_clicks"), + Input("base-path-proj", "value"), +) +def get_base_paths(_, selected_path): + return get_base_path_options(selected_path) + + +@callback( + Output("base-path-proj", "value"), + Input("base-path-proj", "id"), +) +def set_default_base_path(_): + # The log data root is the first option and the default base path + options = get_base_path_options() + return options[0]["value"] if options else None + + @callback( Output("collapse-proj", "is_open"), Input("open-btn-proj", "n_clicks"), diff --git a/dash_app/utils/data_directories.py b/dash_app/utils/data_directories.py index 3547184..a12052c 100644 --- a/dash_app/utils/data_directories.py +++ b/dash_app/utils/data_directories.py @@ -49,3 +49,66 @@ def get_all_root_log_directories(base_path=None) -> tuple[list[str], list[str]]: paths = [os.path.abspath(base_path) + "/"] + directory_paths + file_paths return names, paths + + +def get_base_path_directories( + selected_path: str | None = None, max_depth: int = 1 +) -> tuple[list[str], list[str]]: + """Directories that can be picked as a project base path. + + Lists the log data root and the directories directly below it. When a + directory is already selected, the directories inside it are listed as + well, so that deeper paths are reached one level at a time by selecting a + directory and opening the dropdown again. Names are relative to the log + data root so that they read like the paths a user would type by hand. + """ + root = os.path.abspath(current_app.config["LOG_DATA_PATH"]) + + if not os.path.isdir(root): + return [], [] + + paths = _walk_directories(root, max_depth) + + if selected_path: + selected = os.path.abspath(selected_path) + if selected != root and _is_inside(selected, root): + # The selection and the directories leading to it stay listed, so + # that a deeper selection keeps showing in the dropdown and can be + # stepped back out of. + paths |= _path_and_parents(selected, root) + paths |= _walk_directories(selected, max_depth) + + names = [f"{os.path.basename(root) or root} (root)"] + sorted_paths = [root] + sorted(paths) + names += [os.path.relpath(path, root) for path in sorted_paths[1:]] + + return names, sorted_paths + + +def _walk_directories(base_path: str, max_depth: int) -> set[str]: + directories = set() + + for dirpath, dirnames, _ in os.walk(base_path): + depth = dirpath[len(base_path) :].count(os.sep) + if depth >= max_depth: + dirnames.clear() + continue + + dirnames[:] = [dir for dir in dirnames if not dir.startswith(".")] + directories.update(os.path.join(dirpath, dir) for dir in dirnames) + + return directories + + +def _path_and_parents(path: str, root: str) -> set[str]: + paths = set() + + while path != root and path != os.path.dirname(path): + paths.add(path) + path = os.path.dirname(path) + + return paths + + +def _is_inside(path: str, root: str) -> bool: + return os.path.commonpath([path, root]) == root diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 6cd4b5b..f9a1b32 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -2,6 +2,19 @@ from playwright.sync_api import Page, expect +def select_base_path(page: Page, directory: str): + """Pick a directory in the project form's base path dropdown. + + The options come from a callback and are virtualized, so wait for them to + arrive and type the directory to narrow the list down to the options that + are actually rendered. + """ + page.click("#base-path-proj") + page.locator(".VirtualizedSelectOption").first.wait_for() + page.locator("#base-path-proj input").first.press_sequentially(directory) + page.locator(".VirtualizedSelectOption", has_text=directory).first.click() + + def test_has_title(page: Page): page.goto("http://127.0.0.1:5000/dash/") @@ -24,7 +37,7 @@ def test_can_run_file_count_analysis(page: Page): page.get_by_role("button", name="Create a new project").click() page.get_by_label("Project name").fill("File count test") - page.get_by_label("Base path").fill("./log_data/LO2") + select_base_path(page, "LO2") page.get_by_role("button", name="Create", exact=True).click() project_link = page.locator("#project-group li a", has_text="File count test") @@ -60,7 +73,7 @@ def test_can_run_ano_line_level(page: Page): page.get_by_role("button", name="Create a new project").click() page.get_by_label("Project name").fill("Ano Line Test") - page.get_by_label("Base path").fill("./log_data/LO2") + select_base_path(page, "LO2") page.get_by_role("button", name="Create", exact=True).click() project_link = page.locator("#project-group li a", has_text="Ano Line Test") @@ -106,7 +119,7 @@ def test_error_ano_line_with_bad_inputs(page: Page): page.get_by_role("button", name="Create a new project").click() page.get_by_label("Project name").fill("Error Ano Line") - page.get_by_label("Base path").fill("./log_data/LO2") + select_base_path(page, "LO2") page.get_by_role("button", name="Create", exact=True).click() project_link = page.locator("#project-group li a", has_text="Error Ano Line")