diff --git a/dash_app/callbacks/callback_functions.py b/dash_app/callbacks/callback_functions.py index 763449a..b5a5dcb 100644 --- a/dash_app/callbacks/callback_functions.py +++ b/dash_app/callbacks/callback_functions.py @@ -6,8 +6,14 @@ from dash_app.utils.data_directories import ( get_all_filenames, get_all_root_log_directories, + get_base_path_directories, get_runs, ) +from dash_app.utils.grouping import ( + SEPARATOR, + add_group_column, + get_group_by_options, +) from dash_app.utils.metadata import format_metadata_rows from dash_app.utils.plots import ( create_files_count_plot, @@ -120,7 +126,9 @@ def populate_datatable(analysis_id): return df_dict, columns, metadata_rows, project_id -def create_high_level_plot(switch_on, analysis_id): +def create_high_level_plot( + switch_on, analysis_id, group_by_indices=None, separator=SEPARATOR +): metadata = _fetch_analysis_metadata(analysis_id) analysis_type = metadata.get("analysis_sub_type") analysis_level = metadata.get("analysis_level") @@ -149,6 +157,12 @@ def create_high_level_plot(switch_on, analysis_id): "width": "90%", } + # The parts to group by are read with the same separator they were offered + # with, so the options are built before the data is grouped. + group_by_options = get_group_by_options(df, separator=separator) + if group_by_indices: + df = add_group_column(df, group_by_indices, separator=separator) + if analysis_type == "file-count": fig = create_files_count_plot(df, theme) elif analysis_type == "umap": @@ -162,7 +176,7 @@ def create_high_level_plot(switch_on, analysis_id): metadata_rows = format_metadata_rows(metadata) - return fig, style, metadata_rows, project_id + return fig, style, metadata_rows, project_id, group_by_options def make_api_call(json_payload, endpoint, requests_type="POST"): @@ -223,6 +237,11 @@ def get_log_data_directory_options(project_id: int): return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)] +def get_base_path_options(selected_path=None): + labels, values = get_base_path_directories(selected_path) + return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)] + + def poll_task_status(task_id: int) -> dict: # There shouldn't be any error here unless the task_id is wrong. response, error = make_api_call({}, f"task-status/{task_id}", requests_type="GET") diff --git a/dash_app/components/form_inputs.py b/dash_app/components/form_inputs.py index 770be80..1bacfc1 100644 --- a/dash_app/components/form_inputs.py +++ b/dash_app/components/form_inputs.py @@ -426,6 +426,73 @@ def analysis_name_input(id): ) +def group_by_input(id): + label_id = f"{id}-label" + return dbc.Col( + [ + dbc.Label( + "Color by name part", + id=label_id, + html_for=id, + width="auto", + style={"textDecoration": "underline", "cursor": "pointer"}, + ), + dbc.Tooltip( + "Directory names are split on '_'. Pick the parts to color the plot by, " + "for example the part holding the run type. Picking several parts combines " + "them into one label, in the order they were picked.", + target=label_id, + placement="bottom", + ), + dcc.Dropdown( + id=id, + multi=True, + placeholder="No grouping", + className="dbc border border-light-subtle rounded", + optionHeight=40, + maxHeight=350, + ), + ], + width=6, + ) + + +def separator_input(id): + label_id = f"{id}-label" + return dbc.Col( + [ + dbc.Label( + "Name separator", + id=label_id, + html_for=id, + width="auto", + style={"textDecoration": "underline", "cursor": "pointer"}, + ), + dbc.Tooltip( + "The character the names are split on before the parts are picked. " + "Changing it clears the picked parts, since the parts are counted " + "with the separator they were listed with.", + target=label_id, + placement="bottom", + ), + dcc.Dropdown( + id=id, + options=[ + {"label": "Underscore _", "value": "_"}, + {"label": "Hyphen -", "value": "-"}, + {"label": "Dot .", "value": "."}, + {"label": "Slash /", "value": "/"}, + {"label": "Space", "value": " "}, + ], + value="_", + clearable=False, + className="dbc border border-light-subtle rounded", + ), + ], + width=3, + ) + + def base_path_input(id): label_id = f"{id}-label" return dbc.Col( @@ -438,14 +505,24 @@ def base_path_input(id): style={"textDecoration": "underline", "cursor": "pointer"}, ), dbc.Tooltip( - "Optional. Must be directory inside log_data/. Leave empty or unchanged to use the default. Directory must already exist.", + "The directory the project's log data is read from. Every analysis in the " + "project picks its data from inside this directory, so keep the root to make " + "all of log_data/ available. The directories in log_data/ are listed here; " + "select one and open the list again to see the directories inside it.", target=label_id, placement="bottom", ), - dbc.Input( - id=id, - type="text", - value="./log_data", + dcc.Loading( + type="circle", + delay_show=200, + overlay_style={"visibility": "visible"}, + children=dcc.Dropdown( + id=id, + placeholder="Select a directory under log_data/", + className="dbc border border-light-subtle rounded", + optionHeight=40, + maxHeight=350, + ), ), ], ) diff --git a/dash_app/components/layouts.py b/dash_app/components/layouts.py index e33d911..243b1f5 100644 --- a/dash_app/components/layouts.py +++ b/dash_app/components/layouts.py @@ -2,7 +2,11 @@ from dash import dash_table, dcc, html from dash_app.components.color_mode_switch import color_mode_switch -from dash_app.components.form_inputs import submit_button +from dash_app.components.form_inputs import ( + group_by_input, + separator_input, + submit_button, +) from dash_app.components.forms import project_settings_form from dash_app.components.nav import crate_analysis_nav, nav from dash_app.components.toasts import error_toast, success_toast @@ -429,13 +433,23 @@ def create_project_layout( def create_high_level_viz_result_layout( - plot_content_id, metadata_table_id, error_toast_id, success_toast_id + plot_content_id, + metadata_table_id, + error_toast_id, + success_toast_id, + group_by_id, + separator_id, ): error_toast_row = dbc.Row(error_toast(error_toast_id)) success_toast_row = dbc.Row(success_toast(success_toast_id)) table_row = dbc.Row(dbc.Table(id=metadata_table_id, hover=True, responsive=True)) + group_by_row = dbc.Row( + [group_by_input(group_by_id), separator_input(separator_id)], + class_name="mb-3", + ) + plot_row = dbc.Row( dcc.Loading( type="default", @@ -464,7 +478,7 @@ def create_high_level_viz_result_layout( ) layout = [ - dbc.Container([table_row, error_toast_row, success_toast_row]), + dbc.Container([table_row, group_by_row, error_toast_row, success_toast_row]), dbc.Container(plot_row, fluid=True, style={"paddingBottom": "200px"}), ] diff --git a/dash_app/page_templates/high_level_viz_result_page_base.py b/dash_app/page_templates/high_level_viz_result_page_base.py index f7cfdf0..e288a20 100644 --- a/dash_app/page_templates/high_level_viz_result_page_base.py +++ b/dash_app/page_templates/high_level_viz_result_page_base.py @@ -23,6 +23,8 @@ def create_layout(config, analysis_id=None): config_ids["metadata"], config_ids["error_toast"], config_ids["success_toast"], + config_ids["group_by"], + config_ids["separator"], ) return base + content @@ -30,28 +32,46 @@ def create_layout(config, analysis_id=None): def register_callback(config): config_ids = config["ids"] + # Part numbers only mean something for the separator they were listed with, + # so a new separator starts from no grouping. + @callback( + Output(config_ids["group_by"], "value"), + Input(config_ids["separator"], "value"), + prevent_initial_call=True, + ) + def reset_group_by(_): + return [] + + # The name parts to group by are offered by the plot callback itself, since + # the values to pick from are only known once the results have been read. @callback( Output(config_ids["plot_content"], "figure"), Output(config_ids["plot_content"], "style"), Output(config_ids["metadata"], "children"), Output(config_ids["project_link"], "href"), + Output(config_ids["group_by"], "options"), Output(config_ids["error_toast"], "children"), Output(config_ids["error_toast"], "is_open"), Output(config_ids["success_toast"], "children"), Output(config_ids["success_toast"], "is_open"), Input("switch", "value"), + Input(config_ids["group_by"], "value"), + Input(config_ids["separator"], "value"), State(config_ids["analysis_id"], "data"), ) - def create_plot(switch_on, analysis_id): + def create_plot(switch_on, group_by_indices, separator, analysis_id): try: - fig, style, metadata_rows, project_id = create_high_level_plot( - switch_on, analysis_id + fig, style, metadata_rows, project_id, group_by_options = ( + create_high_level_plot( + switch_on, analysis_id, group_by_indices, separator + ) ) return ( fig, style, [html.Tbody(metadata_rows)], f"/dash/project/{project_id}", + group_by_options, dash.no_update, False, dash.no_update, @@ -63,6 +83,7 @@ def create_plot(switch_on, analysis_id): dash.no_update, dash.no_update, dash.no_update, + dash.no_update, str(e), True, dash.no_update, diff --git a/dash_app/pages/home.py b/dash_app/pages/home.py index 71b711a..9ac050a 100644 --- a/dash_app/pages/home.py +++ b/dash_app/pages/home.py @@ -2,7 +2,7 @@ import dash_bootstrap_components as dbc from dash import ALL, Input, Output, State, callback, dcc -from dash_app.callbacks.callback_functions import make_api_call +from dash_app.callbacks.callback_functions import get_base_path_options, make_api_call from dash_app.components.forms import project_form from dash_app.components.layouts import create_home_layout from dash_app.utils.metadata import format_project_overview @@ -59,6 +59,27 @@ def get_projects(_1, _2): return (group_items, dash.no_update, dash.no_update, dash.no_update, False) +# Selecting a directory reveals the directories below it, so that deeper paths +# can be reached by opening the dropdown again after selecting. +@callback( + Output("base-path-proj", "options"), + Input("open-btn-proj", "n_clicks"), + Input("base-path-proj", "value"), +) +def get_base_paths(_, selected_path): + return get_base_path_options(selected_path) + + +@callback( + Output("base-path-proj", "value"), + Input("base-path-proj", "id"), +) +def set_default_base_path(_): + # The log data root is the first option and the default base path + options = get_base_path_options() + return options[0]["value"] if options else None + + @callback( Output("collapse-proj", "is_open"), Input("open-btn-proj", "n_clicks"), diff --git a/dash_app/pages/result_pages/directory_level_visualisations.py b/dash_app/pages/result_pages/directory_level_visualisations.py index 1c9959c..80d020a 100644 --- a/dash_app/pages/result_pages/directory_level_visualisations.py +++ b/dash_app/pages/result_pages/directory_level_visualisations.py @@ -14,6 +14,8 @@ "metadata": "metadata-high-dir-res", "error_toast": "error-toast-high-dir-res", "success_toast": "success-toast-high-dir-res", + "group_by": "group-by-high-dir-res", + "separator": "separator-high-dir-res", }, } diff --git a/dash_app/pages/result_pages/file_level_visualisations.py b/dash_app/pages/result_pages/file_level_visualisations.py index 20da9ae..e90d018 100644 --- a/dash_app/pages/result_pages/file_level_visualisations.py +++ b/dash_app/pages/result_pages/file_level_visualisations.py @@ -14,6 +14,8 @@ "metadata": "metadata-high-file-res", "error_toast": "error-toast-high-file-res", "success_toast": "success-toast-high-file-res", + "group_by": "group-by-high-file-res", + "separator": "separator-high-file-res", }, } diff --git a/dash_app/utils/data_directories.py b/dash_app/utils/data_directories.py index 3547184..a12052c 100644 --- a/dash_app/utils/data_directories.py +++ b/dash_app/utils/data_directories.py @@ -49,3 +49,66 @@ def get_all_root_log_directories(base_path=None) -> tuple[list[str], list[str]]: paths = [os.path.abspath(base_path) + "/"] + directory_paths + file_paths return names, paths + + +def get_base_path_directories( + selected_path: str | None = None, max_depth: int = 1 +) -> tuple[list[str], list[str]]: + """Directories that can be picked as a project base path. + + Lists the log data root and the directories directly below it. When a + directory is already selected, the directories inside it are listed as + well, so that deeper paths are reached one level at a time by selecting a + directory and opening the dropdown again. Names are relative to the log + data root so that they read like the paths a user would type by hand. + """ + root = os.path.abspath(current_app.config["LOG_DATA_PATH"]) + + if not os.path.isdir(root): + return [], [] + + paths = _walk_directories(root, max_depth) + + if selected_path: + selected = os.path.abspath(selected_path) + if selected != root and _is_inside(selected, root): + # The selection and the directories leading to it stay listed, so + # that a deeper selection keeps showing in the dropdown and can be + # stepped back out of. + paths |= _path_and_parents(selected, root) + paths |= _walk_directories(selected, max_depth) + + names = [f"{os.path.basename(root) or root} (root)"] + sorted_paths = [root] + sorted(paths) + names += [os.path.relpath(path, root) for path in sorted_paths[1:]] + + return names, sorted_paths + + +def _walk_directories(base_path: str, max_depth: int) -> set[str]: + directories = set() + + for dirpath, dirnames, _ in os.walk(base_path): + depth = dirpath[len(base_path) :].count(os.sep) + if depth >= max_depth: + dirnames.clear() + continue + + dirnames[:] = [dir for dir in dirnames if not dir.startswith(".")] + directories.update(os.path.join(dirpath, dir) for dir in dirnames) + + return directories + + +def _path_and_parents(path: str, root: str) -> set[str]: + paths = set() + + while path != root and path != os.path.dirname(path): + paths.add(path) + path = os.path.dirname(path) + + return paths + + +def _is_inside(path: str, root: str) -> bool: + return os.path.commonpath([path, root]) == root diff --git a/dash_app/utils/grouping.py b/dash_app/utils/grouping.py new file mode 100644 index 0000000..0b6826f --- /dev/null +++ b/dash_app/utils/grouping.py @@ -0,0 +1,102 @@ +import polars as pl + +GROUP_COLUMN = "group" +SEPARATOR = "_" + + +def get_group_source_column(df): + """The column naming the points of a plot. + + File level results identify a point by seq_id, which is the directory and + the file name, while directory level results only have the directory name. + """ + return "seq_id" if "seq_id" in df.columns else "run" + + +def add_group_column(df, group_by_indices, source_column=None, separator=SEPARATOR): + """Label every row by the chosen parts of its name. + + Works like LogDelta's group_by_indices setting: the name is split on the + separator and the parts at the given indices, in the order they were given, + make up the label. Negative indices count from the end of the name, so that + the last part can be picked no matter how long the name is. A name that has + none of those parts is used as is. + """ + source_column = source_column or get_group_source_column(df) + + label = ( + pl.col(source_column) + .str.split(separator) + .list.gather(group_by_indices, null_on_oob=True) + .list.drop_nulls() + .list.join(separator) + ) + + return df.with_columns( + pl.when(label.str.len_chars() > 0) + .then(label) + .otherwise(pl.col(source_column)) + .alias(GROUP_COLUMN) + ) + + +def get_group_by_options( + df, source_column=None, separator=SEPARATOR, max_values_shown=4 +): + """Dropdown options for the name parts the results can be grouped by. + + Every option lists the values found at that position, so that the part to + group by can be picked without knowing the naming convention beforehand. + Names that are not all of the same length also get options counted from the + end, where the parts of such names line up. + """ + source_column = source_column or get_group_source_column(df) + if source_column not in df.columns: + return [] + + parts = df[source_column].str.split(separator) + shortest = parts.list.len().min() or 0 + longest = parts.list.len().max() or 0 + + indices = list(range(longest)) + if shortest != longest: + indices += [-position for position in range(1, longest + 1)] + + options = [] + for index in indices: + values = sorted( + set(parts.list.get(index, null_on_oob=True).drop_nulls().to_list()) + ) + if not values: + continue + + label = _part_label(index, shortest, longest) + options.append( + { + "label": f"{label}: {_format_values(values, max_values_shown)}", + "value": index, + } + ) + + return options + + +def _part_label(index, shortest, longest): + if index < 0: + return f"Part {index} (from end)" + + if index == longest - 1 and shortest == longest: + return f"Part {index} (last)" + + return f"Part {index}" + + +def _format_values(values, max_values_shown, max_value_length=24): + shown = ", ".join( + value if len(value) <= max_value_length else f"{value[:max_value_length]}..." + for value in values[:max_values_shown] + ) + if len(values) > max_values_shown: + shown = f"{shown}, ... ({len(values)} values)" + + return shown diff --git a/dash_app/utils/plots.py b/dash_app/utils/plots.py index fb4aa86..44d87f1 100644 --- a/dash_app/utils/plots.py +++ b/dash_app/utils/plots.py @@ -1,6 +1,35 @@ import polars as pl import plotly.graph_objects as go +from dash_app.utils.grouping import GROUP_COLUMN + +# Colors and shapes are combined, so the number of groups that stay apart is the +# count of colors times the count of shapes. The colors are picked from the +# published colour-blind safe palettes (Okabe & Ito, Paul Tol, IBM) as the five +# that stay furthest apart when simulated for protanopia, deuteranopia and +# tritanopia, while keeping enough contrast against both the light and the dark +# theme. The shape carries the difference for anyone who sees no color at all. +GROUP_COLORS = [ + "#33BBEE", # blue + "#E69F00", # orange + "#117733", # green + "#AA3377", # magenta + "#785EF0", # violet +] + +GROUP_SYMBOLS = [ + "circle", + "square", + "diamond", + "triangle-up", + "triangle-down", + "cross", + "x", + "star", + "pentagon", + "hexagram", +] + def get_options(df) -> list[dict]: seq_ids = sorted(df["seq_id"].unique().to_list()) @@ -153,15 +182,13 @@ def create_line_level_plot_minimal( def create_unique_term_count_plot(df, theme="plotly_white"): fig = go.Figure() - fig.add_trace( - go.Scatter( - x=df["unique_term_count"], - y=df["line_count"], - mode="markers", - text=df["run"], - hovertemplate="Run: %{text}
Unique terms: %{x}
Lines:%{y}", - name="Runs", - ) + _add_marker_traces( + fig, + _split_by_group(df, "Runs"), + x_column="unique_term_count", + y_column="line_count", + text_column="run", + hovertemplate="Run: %{text}
Unique terms: %{x}
Lines:%{y}", ) fig.update_layout( @@ -181,32 +208,21 @@ def create_unique_term_count_plot_by_file( ): fig = go.Figure() - if color_by_directory: - runs = df["run"].unique() - for run in sorted(runs): - df_run = df.filter(pl.col("run") == run) - - fig.add_trace( - go.Scatter( - x=df_run["unique_term_count"], - y=df_run["line_count"], - mode="markers", - text=df_run["seq_id"], - hovertemplate="File: %{text}
Unique terms: %{x}
Lines:%{y}", - name=f"Directory: {run}", - ) - ) + if GROUP_COLUMN in df.columns: + frames = _split_by_group(df, "Files") + elif color_by_directory: + frames = _split_by_directory(df) else: - fig.add_trace( - go.Scatter( - x=df["unique_term_count"], - y=df["line_count"], - mode="markers", - text=df["seq_id"], - hovertemplate="File: %{text}
Unique terms: %{x}
Lines:%{y}", - name="Files", - ) - ) + frames = [("Files", df)] + + _add_marker_traces( + fig, + frames, + x_column="unique_term_count", + y_column="line_count", + text_column="seq_id", + hovertemplate="File: %{text}
Unique terms: %{x}
Lines:%{y}", + ) fig.update_layout( title="Unique term count by file", @@ -223,15 +239,13 @@ def create_unique_term_count_plot_by_file( def create_files_count_plot(df, theme="plotly_white"): fig = go.Figure() - fig.add_trace( - go.Scatter( - x=df["file_count"], - y=df["line_count"], - mode="markers", - text=df["run"], - hovertemplate="Run: %{text}
Files: %{x}
Lines:%{y}", - name="Runs", - ) + _add_marker_traces( + fig, + _split_by_group(df, "Runs"), + x_column="file_count", + y_column="line_count", + text_column="run", + hovertemplate="Run: %{text}
Files: %{x}
Lines:%{y}", ) fig.update_layout( @@ -241,38 +255,30 @@ def create_files_count_plot(df, theme="plotly_white"): template=theme, ) + fig.update_yaxes(type="log") + return fig def create_umap_plot(df, group_col, color_by_directory=False, theme="plotly_white"): fig = go.Figure() - if group_col == "seq_id" and color_by_directory: - runs = df["run"].unique() - for run in sorted(runs): - df_run = df.filter(pl.col("run") == run) - - fig.add_trace( - go.Scatter( - x=df_run["UMAP1"], - y=df_run["UMAP2"], - mode="markers", - text=df_run[group_col], - hovertemplate=f"{group_col}: %{{text}}
UMAP1: %{{x}}
UMAP2:%{{y}}", - name=f"Directory: {run}", - marker=dict(symbol="x", size=4), - ) - ) + if GROUP_COLUMN in df.columns: + frames = _split_by_group(df, None) + elif group_col == "seq_id" and color_by_directory: + frames = _split_by_directory(df) else: - fig.add_trace( - go.Scatter( - x=df["UMAP1"], - y=df["UMAP2"], - mode="markers", - text=df[group_col], - hovertemplate=f"{group_col}: %{{text}}
UMAP1: %{{x}}
UMAP2:%{{y}}", - ) - ) + frames = [(None, df)] + + _add_marker_traces( + fig, + frames, + x_column="UMAP1", + y_column="UMAP2", + text_column=group_col, + hovertemplate=f"{group_col}: %{{text}}
UMAP1: %{{x}}
UMAP2:%{{y}}", + size=6, + ) fig.update_layout( title="UMAP comparison", @@ -284,6 +290,54 @@ def create_umap_plot(df, group_col, color_by_directory=False, theme="plotly_whit return fig +def _add_marker_traces( + fig, frames, x_column, y_column, text_column, hovertemplate, size=8 +): + """Draw one marker trace per frame, each with its own color and shape.""" + for index, (name, frame) in enumerate(frames): + fig.add_trace( + go.Scatter( + x=frame[x_column], + y=frame[y_column], + mode="markers", + text=frame[text_column], + hovertemplate=hovertemplate, + name=name, + marker=_group_marker(index, size), + ) + ) + + +def _group_marker(index, size): + # The colors are cycled through first and the shape changes once they run + # out, so every combination is used before any of them comes back. + return dict( + color=GROUP_COLORS[index % len(GROUP_COLORS)], + symbol=GROUP_SYMBOLS[(index // len(GROUP_COLORS)) % len(GROUP_SYMBOLS)], + size=size, + # A neutral outline keeps the palest markers visible in both themes. + line=dict(width=1, color="rgba(128, 128, 128, 0.8)"), + ) + + +def _split_by_group(df, default_name): + """Frames to draw as separate traces, one per group when the data is grouped.""" + if GROUP_COLUMN not in df.columns: + return [(default_name, df)] + + return [ + (group, df.filter(pl.col(GROUP_COLUMN) == group)) + for group in sorted(df[GROUP_COLUMN].unique()) + ] + + +def _split_by_directory(df): + return [ + (f"Directory: {run}", df.filter(pl.col("run") == run)) + for run in sorted(df["run"].unique()) + ] + + def _wrap_log(text, width=80): return "
".join([text[i : i + width] for i in range(0, len(text), width)]) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 67120d5..2237235 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -26,6 +26,8 @@ services: command: celery -A server.make_celery worker --concurrency=4 --loglevel=info volumes: - .:/app + - ${LOG_DATA_DIRECTORY}:/app/log_data/ + - ${RESULTS_DIRECTORY}:/app/analysis_results environment: DB_URL: "${DB_URL}" CELERY_BROKER_URL: "${CELERY_BROKER_URL}" diff --git a/docs/RunAgainstLogLeadBranch.md b/docs/RunAgainstLogLeadBranch.md new file mode 100644 index 0000000..43ca9fe --- /dev/null +++ b/docs/RunAgainstLogLeadBranch.md @@ -0,0 +1,102 @@ +# Testing VisualLogAnalyzer against an unreleased LogLead branch + +Use this when you want to test LogLead changes on GitHub (a branch not yet +released to PyPI) against VisualLogAnalyzer. + +## 1. Point requirements.txt at the branch + +Edit [`requirements.txt`](../requirements.txt) and replace the pinned line +``` +LogLead==1.2.3 +``` +with a direct URL to a source tarball of the branch: + +``` +LogLead @ https://github.com/EvoTestOps/LogLead/archive/refs/heads/.tar.gz +``` + +Swap `` for whichever branch you're testing, e.g. `feature/more-loaders`. + +Note: this uses a plain HTTPS tarball, not a `git+https://...` reference — +a `git+` URL would need the `git` binary inside the build image, which the +`python:3.12-slim` base image doesn't include, and we don't want a +permanent Dockerfile change just for occasional branch testing. + +## 2. Rebuild the Docker images + +LogLead is installed at image build time (`Dockerfile` runs +`pip install -r requirements.txt`), so a container restart alone won't +pick up the change — you need to rebuild: + +```bash +docker compose -f docker-compose-dev.yml build app celery +docker compose -f docker-compose-dev.yml up -d +``` + +(use `docker-compose.yml` instead of `docker-compose-dev.yml` if you're +not using the dev override). + +If you re-run this later to pick up *new* commits pushed to the same +branch, Docker's layer cache won't notice the branch moved (the +requirements.txt text is unchanged), so force it: + +```bash +docker compose -f docker-compose-dev.yml build --no-cache app celery +``` + +## 3. Verify what actually got installed + +```bash +docker compose -f docker-compose-dev.yml exec app pip freeze | grep -i loglead +``` + +You should see the tarball URL you set in step 1, confirming the branch +version installed rather than the PyPI release, e.g.: + +``` +LogLead @ https://github.com/EvoTestOps/LogLead/archive/refs/heads/feature/more-loaders.tar.gz +``` + +## 4. Automated smoke test + +Tests need the repo's bundled `log_data/LO2` mounted at `/app/log_data`, so (re)create the containers with `.env.sample` (`LOG_DATA_DIRECTORY=./log_data`) rather than your own `.env` — a plain `exec` on an already-running container won't change its mounts: + +```bash +docker compose -f docker-compose-dev.yml --env-file .env.sample up -d --force-recreate app celery +docker compose -f docker-compose-dev.yml exec app pytest tests +curl -s http://localhost:5000/health +``` + +## 5. Switch back to your own data + +Recreate `app`/`celery` again, this time without `--env-file .env.sample`, so they pick up your regular `.env` (e.g. `LOG_DATA_DIRECTORY=~/Datasets`) instead: + +```bash +docker compose -f docker-compose-dev.yml up -d --force-recreate app celery +``` + +## 6. Running and Manual Testing + +This repeats the relevant parts of [usage_guide.md](usage_guide.md) — see +that file for the full walkthrough and screenshots. + +1. Make sure `LOG_DATA_DIRECTORY` and `RESULTS_DIRECTORY` in `.env` (or + `.env.sample`) point at the correct folders: + ``` + LOG_DATA_DIRECTORY=./log_data + RESULTS_DIRECTORY=./analysis_results + ``` +2. Start the stack (already done in step 2 above if you rebuilt with + `up -d`; otherwise `docker compose -f docker-compose-dev.yml --env-file + .env up`). +3. Open in a browser. + +## 7. Revert back to the released version when done + +``` +LogLead==1.2.3 +``` + +then rebuild again (step 2). Don't leave a branch pin committed to `main` +long-term — it's a floating target (CI and anyone else building the image +would silently pick up whatever the branch currently contains). diff --git a/docs/usage_guide.md b/docs/usage_guide.md index 28b163e..b5f1192 100644 --- a/docs/usage_guide.md +++ b/docs/usage_guide.md @@ -19,6 +19,51 @@ To omit the `--env-file` flag, rename `.env.sample` to `.env`. For example with Building the application for the first time may take 1-2 minutes. Once the applitcation is running navigate to to access the homepage. +## Stopping and restarting + +The commands below assume you renamed `.env.sample` to `.env`. If you did not, add `--env-file .env.sample` to *every* `docker compose` command, not just `up` — compose reads the env file each time to resolve `${LOG_DATA_DIRECTORY}` and the other variables. Likewise, if you are using the development compose file, repeat `-f docker-compose-dev.yml` on every command. + +### Stopping + +If you started the application in the foreground with `docker compose up`, press `Ctrl+C` in that terminal. To stop it from another terminal, or when it was started in the background, use one of: + +``` +docker compose stop # stop the containers but keep them, for a fast restart +docker compose down # stop and remove the containers +``` + +Both are safe to run: your projects and analyses are stored in the `db_data` Docker volume and the results in `analysis_results/`, and neither command touches them. + +### Restarting + +``` +docker compose start # after `docker compose stop` +docker compose up -d # after `docker compose down`, or any time; -d runs it in the background +docker compose restart # stop and start in one go, e.g. to clear a stuck analysis worker +``` + +Database migrations are applied automatically every time the app container starts, so no extra step is needed after an update. + +With `-d` the application runs in the background and your terminal stays free. Use `docker compose ps` to see what is running and `docker compose logs -f` to follow the output (add a service name, e.g. `docker compose logs -f celery`, to follow just one). + +### Restarting after changing the code or configuration + +Changes to the `.env` file take effect when the containers are recreated: `docker compose up -d`. Changes to the application code or to `requirements.txt` require the images to be rebuilt first: + +``` +docker compose up -d --build +``` + +(With the development compose file the code is mounted into the containers, so only `requirements.txt` changes need a rebuild there.) + +### Starting over + +``` +docker compose down -v +``` + +The `-v` flag additionally deletes the `db_data` volume, which removes **all** projects and analysis records. The result files in `analysis_results/` are not deleted, but they are left orphaned — delete its contents as well if you want a completely clean state. The database schema is recreated automatically on the next start. + ## Running analyses The repository includes an example light-oauth-2 dataset. It contains a `Labeled` directory which has known cases (either correct or some type of error), and a `Hidden_Group_1` which contains unknown cases. Reviewing the dataset structure is recommended. diff --git a/server/analysis/log_analysis_pipeline.py b/server/analysis/log_analysis_pipeline.py index 267cf4a..7de94d2 100644 --- a/server/analysis/log_analysis_pipeline.py +++ b/server/analysis/log_analysis_pipeline.py @@ -9,6 +9,12 @@ aggregate_file_level_with_file_names, ) +RUN_COLUMN = "run" + +# The hold-out scores the test data in several batches, so the rows are tagged +# with their original position and put back in that order afterwards. +ROW_ORDER_COLUMN = "__row_order" + class ManualTrainTestPipeline: def __init__( @@ -78,10 +84,66 @@ def _enhance_test_train(self, df): return enhancer.df def analyze(self): + self._results = self._analyze_holding_out_target_run( + self._df_train, self._df_test + ) + + def _analyze_holding_out_target_run(self, df_train, df_test) -> pl.DataFrame: + """Score each test run against training data that does not contain that run. + + Comparing runs against each other means giving the same directory as both the + train and the test data, which would otherwise leave every run in the training + data used to score it. That leaks the answer into the scores: the OOV detector + cannot find an unknown word in a run whose words it has just learned, and a run + can end up alone in a KMeans cluster centred on itself, a distance of exactly 0 + from it. LogDelta compares each run against the other runs only, and so do we. + """ analyzer = LogAnalyzer(item_list_col=self._item_list_col) - analyzer.manual_train_split(self._df_train, self._df_test, self._vectorizer) - self._results = analyzer.run_models(self._model_names) + df_test = df_test.with_row_index(ROW_ORDER_COLUMN) + train_runs = set(df_train[RUN_COLUMN].unique().to_list()) + test_runs = df_test[RUN_COLUMN].unique(maintain_order=True).to_list() + + results = [] + + # Runs the training data does not contain are not leaking anything, so one + # model trained on all of it serves all of them. + shared_runs = [run for run in test_runs if run not in train_runs] + if shared_runs: + analyzer.manual_train_split( + df_train, + df_test.filter(pl.col(RUN_COLUMN).is_in(shared_runs)), + self._vectorizer, + ) + results.append(analyzer.run_models(self._model_names)) + + for run in [run for run in test_runs if run in train_runs]: + df_train_without_run = df_train.filter(pl.col(RUN_COLUMN) != run) + + # Nothing is left to compare the run against, which happens when it is + # the only run in the training data. Its rows get no scores. + if df_train_without_run.is_empty(): + continue + + analyzer.manual_train_split( + df_train_without_run, + df_test.filter(pl.col(RUN_COLUMN) == run), + self._vectorizer, + ) + results.append(analyzer.run_models(self._model_names)) + + if not results: + raise ValueError( + "No comparison data left after holding out the run being scored. " + "Anomaly detection needs training data from at least one run other " + "than the one under test." + ) + + return ( + pl.concat(results, how="vertical") + .sort(ROW_ORDER_COLUMN) + .drop(ROW_ORDER_COLUMN) + ) def _analyze_grouped_by_file( self, df_train, df_test, common_file_names: list[str] @@ -89,15 +151,14 @@ def _analyze_grouped_by_file( if not common_file_names or len(common_file_names) == 0: raise ValueError("No common file names found. Try changing settings.") - analyzer = LogAnalyzer(item_list_col=self._item_list_col) - results = [] for file_name in common_file_names: train_subset = df_train.filter(pl.col("file_name") == file_name) test_subset = df_test.filter(pl.col("file_name") == file_name) - analyzer.manual_train_split(train_subset, test_subset, self._vectorizer) - results.append(analyzer.run_models(self._model_names)) + results.append( + self._analyze_holding_out_target_run(train_subset, test_subset) + ) return pl.concat(results, how="vertical") diff --git a/server/analysis/utils/file_level_analysis.py b/server/analysis/utils/file_level_analysis.py index fd699f5..4e218b4 100644 --- a/server/analysis/utils/file_level_analysis.py +++ b/server/analysis/utils/file_level_analysis.py @@ -31,12 +31,16 @@ def aggregate_file_level(df, item_list_col, mask_type=None): col_dtype = df.select(pl.col(item_list_col)).dtypes[0] + # The run each file belongs to is carried along so that anomaly detection can + # hold the run being scored out of its own training data. if isinstance(col_dtype, pl.List): - df = df.select("seq_id", item_list_col).explode(item_list_col) + df = df.select("seq_id", "run", item_list_col).explode(item_list_col) else: - df = df.select("seq_id", item_list_col) + df = df.select("seq_id", "run", item_list_col) - df = (df.group_by("seq_id").agg(pl.col(item_list_col))).sort("seq_id") + df = ( + df.group_by("seq_id").agg([pl.col(item_list_col), pl.col("run").first()]) + ).sort("seq_id") return df @@ -49,12 +53,16 @@ def aggregate_file_level_with_file_names(df, item_list_col): col_dtype = df.select(pl.col(item_list_col)).dtypes[0] if isinstance(col_dtype, pl.List): - df = df.select("seq_id", "file_name", item_list_col).explode(item_list_col) + df = df.select("seq_id", "file_name", "run", item_list_col).explode( + item_list_col + ) else: - df = df.select("seq_id", "file_name", item_list_col) + df = df.select("seq_id", "file_name", "run", item_list_col) - df = (df.group_by(["seq_id", "file_name"]).agg(pl.col(item_list_col))).sort( - "seq_id" - ) + df = ( + df.group_by(["seq_id", "file_name"]).agg( + [pl.col(item_list_col), pl.col("run").first()] + ) + ).sort("seq_id") return df diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 6cd4b5b..f9a1b32 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -2,6 +2,19 @@ from playwright.sync_api import Page, expect +def select_base_path(page: Page, directory: str): + """Pick a directory in the project form's base path dropdown. + + The options come from a callback and are virtualized, so wait for them to + arrive and type the directory to narrow the list down to the options that + are actually rendered. + """ + page.click("#base-path-proj") + page.locator(".VirtualizedSelectOption").first.wait_for() + page.locator("#base-path-proj input").first.press_sequentially(directory) + page.locator(".VirtualizedSelectOption", has_text=directory).first.click() + + def test_has_title(page: Page): page.goto("http://127.0.0.1:5000/dash/") @@ -24,7 +37,7 @@ def test_can_run_file_count_analysis(page: Page): page.get_by_role("button", name="Create a new project").click() page.get_by_label("Project name").fill("File count test") - page.get_by_label("Base path").fill("./log_data/LO2") + select_base_path(page, "LO2") page.get_by_role("button", name="Create", exact=True).click() project_link = page.locator("#project-group li a", has_text="File count test") @@ -60,7 +73,7 @@ def test_can_run_ano_line_level(page: Page): page.get_by_role("button", name="Create a new project").click() page.get_by_label("Project name").fill("Ano Line Test") - page.get_by_label("Base path").fill("./log_data/LO2") + select_base_path(page, "LO2") page.get_by_role("button", name="Create", exact=True).click() project_link = page.locator("#project-group li a", has_text="Ano Line Test") @@ -106,7 +119,7 @@ def test_error_ano_line_with_bad_inputs(page: Page): page.get_by_role("button", name="Create a new project").click() page.get_by_label("Project name").fill("Error Ano Line") - page.get_by_label("Base path").fill("./log_data/LO2") + select_base_path(page, "LO2") page.get_by_role("button", name="Create", exact=True).click() project_link = page.locator("#project-group li a", has_text="Error Ano Line") diff --git a/tests/test_log_analysis_pipeline.py b/tests/test_log_analysis_pipeline.py new file mode 100644 index 0000000..2696bab --- /dev/null +++ b/tests/test_log_analysis_pipeline.py @@ -0,0 +1,140 @@ +import polars as pl +import pytest + +import server.analysis.log_analysis_pipeline as pipeline_module +from server.analysis.log_analysis_pipeline import ManualTrainTestPipeline +from server.analysis.log_analyzer import LogAnalyzer +from server.analysis.utils.analysis_helpers import create_vectorizer + + +def make_df(runs: dict[str, str]) -> pl.DataFrame: + """Log lines for the given runs, each run holding one word only it has.""" + rows = [ + { + "run": run, + "file_name": file_name, + "seq_id": f"{run}_{file_name}", + "line_number": line, + "e_words": ["shared", "words", f"line{line}"] + ([own_word] if line else []), + } + for run, own_word in runs.items() + for file_name in ("a", "b") + for line in range(4) + ] + return pl.DataFrame(rows) + + +def make_pipeline(df_train, df_test, models=None): + pipeline = ManualTrainTestPipeline( + model_names=models or ["oovd"], + item_list_col="e_words", + vectorizer=create_vectorizer("count"), + ) + pipeline._df_train = df_train + pipeline._df_test = df_test + return pipeline + + +@pytest.fixture +def recorded_train_splits(monkeypatch): + """The (train, test) frames handed to the analyzer for each model fit.""" + splits = [] + + class RecordingLogAnalyzer(LogAnalyzer): + def manual_train_split(self, train_df, test_df, vectorizer): + splits.append((train_df, test_df)) + super().manual_train_split(train_df, test_df, vectorizer) + + monkeypatch.setattr(pipeline_module, "LogAnalyzer", RecordingLogAnalyzer) + return splits + + +class TestTargetRunHoldOut: + def test_target_run_is_never_in_its_own_training_data(self, recorded_train_splits): + df = make_df({"run_1": "alpha", "run_2": "beta", "run_3": "gamma"}) + + pipeline = make_pipeline(df, df) + pipeline.aggregate_to_run_level() + pipeline.analyze() + + assert recorded_train_splits + for train_df, test_df in recorded_train_splits: + scored_runs = set(test_df["run"].to_list()) + training_runs = set(train_df["run"].to_list()) + assert not scored_runs & training_runs + + def test_oov_detector_finds_the_words_only_the_scored_run_has(self): + df = make_df({"run_1": "alpha", "run_2": "beta", "run_3": "gamma"}) + + pipeline = make_pipeline(df, df) + pipeline.aggregate_to_run_level() + pipeline.analyze() + + # Without the hold-out every run would score 0: its own words would have + # been part of the vocabulary it is compared against. + assert pipeline.results.filter(pl.col("oovd_pred_ano_proba") == 0).is_empty() + + def test_results_keep_the_order_of_the_test_data(self): + df = make_df({"run_1": "alpha", "run_2": "beta", "run_3": "gamma"}) + + pipeline = make_pipeline(df, df) + pipeline.aggregate_to_run_level() + pipeline.analyze() + + assert pipeline.results["run"].to_list() == ["run_1", "run_2", "run_3"] + + def test_runs_missing_from_the_training_data_share_one_model( + self, recorded_train_splits + ): + df_train = make_df({"train_1": "alpha", "train_2": "beta"}) + df_test = make_df({"test_1": "gamma", "test_2": "delta"}) + + pipeline = make_pipeline(df_train, df_test) + pipeline.aggregate_to_run_level() + pipeline.analyze() + + assert len(recorded_train_splits) == 1 + train_df, test_df = recorded_train_splits[0] + assert sorted(test_df["run"].to_list()) == ["test_1", "test_2"] + assert sorted(train_df["run"].unique().to_list()) == ["train_1", "train_2"] + + def test_only_the_overlapping_runs_get_their_own_model( + self, recorded_train_splits + ): + df_train = make_df({"run_1": "alpha", "run_2": "beta"}) + df_test = make_df({"run_2": "beta", "run_3": "gamma"}) + + pipeline = make_pipeline(df_train, df_test) + pipeline.aggregate_to_run_level() + pipeline.analyze() + + # One model for run_3, which the training data does not contain, and one + # trained without run_2 to score run_2. + assert len(recorded_train_splits) == 2 + held_out = [ + (sorted(test_df["run"].to_list()), sorted(train_df["run"].to_list())) + for train_df, test_df in recorded_train_splits + ] + assert (["run_3"], ["run_1", "run_2"]) in held_out + assert (["run_2"], ["run_1"]) in held_out + + def test_file_level_holds_out_the_whole_run(self, recorded_train_splits): + df = make_df({"run_1": "alpha", "run_2": "beta"}) + + pipeline = make_pipeline(df, df) + pipeline.aggregate_to_file_level() + pipeline.analyze() + + for train_df, test_df in recorded_train_splits: + scored_runs = set(test_df["run"].to_list()) + training_runs = set(train_df["run"].to_list()) + assert not scored_runs & training_runs + + def test_single_run_on_both_sides_raises(self): + df = make_df({"only_run": "alpha"}) + + pipeline = make_pipeline(df, df) + pipeline.aggregate_to_run_level() + + with pytest.raises(ValueError, match="No comparison data left"): + pipeline.analyze()