Feature/coloring

dash_app/callbacks/callback_functions.py

@@ -6,8 +6,14 @@
 from dash_app.utils.data_directories import (
     get_all_filenames,
     get_all_root_log_directories,
+    get_base_path_directories,
     get_runs,
 )
+from dash_app.utils.grouping import (
+    SEPARATOR,
+    add_group_column,
+    get_group_by_options,
+)
 from dash_app.utils.metadata import format_metadata_rows
 from dash_app.utils.plots import (
     create_files_count_plot,
@@ -120,7 +126,9 @@ def populate_datatable(analysis_id):
     return df_dict, columns, metadata_rows, project_id
 
 
-def create_high_level_plot(switch_on, analysis_id):
+def create_high_level_plot(
+    switch_on, analysis_id, group_by_indices=None, separator=SEPARATOR
+):
     metadata = _fetch_analysis_metadata(analysis_id)
     analysis_type = metadata.get("analysis_sub_type")
     analysis_level = metadata.get("analysis_level")
@@ -149,6 +157,12 @@ def create_high_level_plot(switch_on, analysis_id):
         "width": "90%",
     }
 
+    # The parts to group by are read with the same separator they were offered
+    # with, so the options are built before the data is grouped.
+    group_by_options = get_group_by_options(df, separator=separator)
+    if group_by_indices:
+        df = add_group_column(df, group_by_indices, separator=separator)
+
     if analysis_type == "file-count":
         fig = create_files_count_plot(df, theme)
     elif analysis_type == "umap":
@@ -162,7 +176,7 @@ def create_high_level_plot(switch_on, analysis_id):
 
     metadata_rows = format_metadata_rows(metadata)
 
-    return fig, style, metadata_rows, project_id
+    return fig, style, metadata_rows, project_id, group_by_options
 
 
 def make_api_call(json_payload, endpoint, requests_type="POST"):
@@ -223,6 +237,11 @@ def get_log_data_directory_options(project_id: int):
     return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)]
 
 
+def get_base_path_options(selected_path=None):
+    labels, values = get_base_path_directories(selected_path)
+    return [{"label": lab, "value": val} for (lab, val) in zip(labels, values)]
+
+
 def poll_task_status(task_id: int) -> dict:
     # There shouldn't be any error here unless the task_id is wrong.
     response, error = make_api_call({}, f"task-status/{task_id}", requests_type="GET")

dash_app/components/form_inputs.py

@@ -426,6 +426,73 @@ def analysis_name_input(id):
     )
 
 
+def group_by_input(id):
+    label_id = f"{id}-label"
+    return dbc.Col(
+        [
+            dbc.Label(
+                "Color by name part",
+                id=label_id,
+                html_for=id,
+                width="auto",
+                style={"textDecoration": "underline", "cursor": "pointer"},
+            ),
+            dbc.Tooltip(
+                "Directory names are split on '_'. Pick the parts to color the plot by, "
+                "for example the part holding the run type. Picking several parts combines "
+                "them into one label, in the order they were picked.",
+                target=label_id,
+                placement="bottom",
+            ),
+            dcc.Dropdown(
+                id=id,
+                multi=True,
+                placeholder="No grouping",
+                className="dbc border border-light-subtle rounded",
+                optionHeight=40,
+                maxHeight=350,
+            ),
+        ],
+        width=6,
+    )
+
+
+def separator_input(id):
+    label_id = f"{id}-label"
+    return dbc.Col(
+        [
+            dbc.Label(
+                "Name separator",
+                id=label_id,
+                html_for=id,
+                width="auto",
+                style={"textDecoration": "underline", "cursor": "pointer"},
+            ),
+            dbc.Tooltip(
+                "The character the names are split on before the parts are picked. "
+                "Changing it clears the picked parts, since the parts are counted "
+                "with the separator they were listed with.",
+                target=label_id,
+                placement="bottom",
+            ),
+            dcc.Dropdown(
+                id=id,
+                options=[
+                    {"label": "Underscore  _", "value": "_"},
+                    {"label": "Hyphen  -", "value": "-"},
+                    {"label": "Dot  .", "value": "."},
+                    {"label": "Slash  /", "value": "/"},
+                    {"label": "Space", "value": " "},
+                ],
+                value="_",
+                clearable=False,
+                className="dbc border border-light-subtle rounded",
+            ),
+        ],
+        width=3,
+    )
+
+
 def base_path_input(id):
     label_id = f"{id}-label"
     return dbc.Col(
@@ -438,14 +505,24 @@ def base_path_input(id):
                 style={"textDecoration": "underline", "cursor": "pointer"},
             ),
             dbc.Tooltip(
-                "Optional. Must be directory inside log_data/. Leave empty or unchanged to use the default. Directory must already exist.",
+                "The directory the project's log data is read from. Every analysis in the "
+                "project picks its data from inside this directory, so keep the root to make "
+                "all of log_data/ available. The directories in log_data/ are listed here; "
+                "select one and open the list again to see the directories inside it.",
                 target=label_id,
                 placement="bottom",
             ),
-            dbc.Input(
-                id=id,
-                type="text",
-                value="./log_data",
+            dcc.Loading(
+                type="circle",
+                delay_show=200,
+                overlay_style={"visibility": "visible"},
+                children=dcc.Dropdown(
+                    id=id,
+                    placeholder="Select a directory under log_data/",
+                    className="dbc border border-light-subtle rounded",
+                    optionHeight=40,
+                    maxHeight=350,
+                ),
             ),
         ],
     )

dash_app/components/layouts.py

@@ -2,7 +2,11 @@
 from dash import dash_table, dcc, html
 
 from dash_app.components.color_mode_switch import color_mode_switch
-from dash_app.components.form_inputs import submit_button
+from dash_app.components.form_inputs import (
+    group_by_input,
+    separator_input,
+    submit_button,
+)
 from dash_app.components.forms import project_settings_form
 from dash_app.components.nav import crate_analysis_nav, nav
 from dash_app.components.toasts import error_toast, success_toast
@@ -429,13 +433,23 @@ def create_project_layout(
 
 
 def create_high_level_viz_result_layout(
-    plot_content_id, metadata_table_id, error_toast_id, success_toast_id
+    plot_content_id,
+    metadata_table_id,
+    error_toast_id,
+    success_toast_id,
+    group_by_id,
+    separator_id,
 ):
     error_toast_row = dbc.Row(error_toast(error_toast_id))
     success_toast_row = dbc.Row(success_toast(success_toast_id))
 
     table_row = dbc.Row(dbc.Table(id=metadata_table_id, hover=True, responsive=True))
 
+    group_by_row = dbc.Row(
+        [group_by_input(group_by_id), separator_input(separator_id)],
+        class_name="mb-3",
+    )
+
     plot_row = dbc.Row(
         dcc.Loading(
             type="default",
@@ -464,7 +478,7 @@ def create_high_level_viz_result_layout(
     )
 
     layout = [
-        dbc.Container([table_row, error_toast_row, success_toast_row]),
+        dbc.Container([table_row, group_by_row, error_toast_row, success_toast_row]),
         dbc.Container(plot_row, fluid=True, style={"paddingBottom": "200px"}),
     ]
 

dash_app/page_templates/high_level_viz_result_page_base.py

@@ -23,35 +23,55 @@ def create_layout(config, analysis_id=None):
         config_ids["metadata"],
         config_ids["error_toast"],
         config_ids["success_toast"],
+        config_ids["group_by"],
+        config_ids["separator"],
     )
     return base + content
 
 
 def register_callback(config):
     config_ids = config["ids"]
 
+    # Part numbers only mean something for the separator they were listed with,
+    # so a new separator starts from no grouping.
+    @callback(
+        Output(config_ids["group_by"], "value"),
+        Input(config_ids["separator"], "value"),
+        prevent_initial_call=True,
+    )
+    def reset_group_by(_):
+        return []
+
+    # The name parts to group by are offered by the plot callback itself, since
+    # the values to pick from are only known once the results have been read.
     @callback(
         Output(config_ids["plot_content"], "figure"),
         Output(config_ids["plot_content"], "style"),
         Output(config_ids["metadata"], "children"),
         Output(config_ids["project_link"], "href"),
+        Output(config_ids["group_by"], "options"),
         Output(config_ids["error_toast"], "children"),
         Output(config_ids["error_toast"], "is_open"),
         Output(config_ids["success_toast"], "children"),
         Output(config_ids["success_toast"], "is_open"),
         Input("switch", "value"),
+        Input(config_ids["group_by"], "value"),
+        Input(config_ids["separator"], "value"),
         State(config_ids["analysis_id"], "data"),
     )
-    def create_plot(switch_on, analysis_id):
+    def create_plot(switch_on, group_by_indices, separator, analysis_id):
         try:
-            fig, style, metadata_rows, project_id = create_high_level_plot(
-                switch_on, analysis_id
+            fig, style, metadata_rows, project_id, group_by_options = (
+                create_high_level_plot(
+                    switch_on, analysis_id, group_by_indices, separator
+                )
             )
             return (
                 fig,
                 style,
                 [html.Tbody(metadata_rows)],
                 f"/dash/project/{project_id}",
+                group_by_options,
                 dash.no_update,
                 False,
                 dash.no_update,
@@ -63,6 +83,7 @@ def create_plot(switch_on, analysis_id):
                 dash.no_update,
                 dash.no_update,
                 dash.no_update,
+                dash.no_update,
                 str(e),
                 True,
                 dash.no_update,

dash_app/pages/home.py

@@ -2,7 +2,7 @@
 import dash_bootstrap_components as dbc
 from dash import ALL, Input, Output, State, callback, dcc
 
-from dash_app.callbacks.callback_functions import make_api_call
+from dash_app.callbacks.callback_functions import get_base_path_options, make_api_call
 from dash_app.components.forms import project_form
 from dash_app.components.layouts import create_home_layout
 from dash_app.utils.metadata import format_project_overview
@@ -59,6 +59,27 @@ def get_projects(_1, _2):
     return (group_items, dash.no_update, dash.no_update, dash.no_update, False)
 
 
+# Selecting a directory reveals the directories below it, so that deeper paths
+# can be reached by opening the dropdown again after selecting.
+@callback(
+    Output("base-path-proj", "options"),
+    Input("open-btn-proj", "n_clicks"),
+    Input("base-path-proj", "value"),
+)
+def get_base_paths(_, selected_path):
+    return get_base_path_options(selected_path)
+
+
+@callback(
+    Output("base-path-proj", "value"),
+    Input("base-path-proj", "id"),
+)
+def set_default_base_path(_):
+    # The log data root is the first option and the default base path
+    options = get_base_path_options()
+    return options[0]["value"] if options else None
+
+
 @callback(
     Output("collapse-proj", "is_open"),
     Input("open-btn-proj", "n_clicks"),

dash_app/pages/result_pages/directory_level_visualisations.py

@@ -14,6 +14,8 @@
         "metadata": "metadata-high-dir-res",
         "error_toast": "error-toast-high-dir-res",
         "success_toast": "success-toast-high-dir-res",
+        "group_by": "group-by-high-dir-res",
+        "separator": "separator-high-dir-res",
     },
 }
 

dash_app/pages/result_pages/file_level_visualisations.py

@@ -14,6 +14,8 @@
         "metadata": "metadata-high-file-res",
         "error_toast": "error-toast-high-file-res",
         "success_toast": "success-toast-high-file-res",
+        "group_by": "group-by-high-file-res",
+        "separator": "separator-high-file-res",
     },
 }
 

dash_app/utils/data_directories.py

@@ -49,3 +49,66 @@ def get_all_root_log_directories(base_path=None) -> tuple[list[str], list[str]]:
     paths = [os.path.abspath(base_path) + "/"] + directory_paths + file_paths
 
     return names, paths
+
+
+def get_base_path_directories(
+    selected_path: str | None = None, max_depth: int = 1
+) -> tuple[list[str], list[str]]:
+    """Directories that can be picked as a project base path.
+
+    Lists the log data root and the directories directly below it. When a
+    directory is already selected, the directories inside it are listed as
+    well, so that deeper paths are reached one level at a time by selecting a
+    directory and opening the dropdown again. Names are relative to the log
+    data root so that they read like the paths a user would type by hand.
+    """
+    root = os.path.abspath(current_app.config["LOG_DATA_PATH"])
+
+    if not os.path.isdir(root):
+        return [], []
+
+    paths = _walk_directories(root, max_depth)
+
+    if selected_path:
+        selected = os.path.abspath(selected_path)
+        if selected != root and _is_inside(selected, root):
+            # The selection and the directories leading to it stay listed, so
+            # that a deeper selection keeps showing in the dropdown and can be
+            # stepped back out of.
+            paths |= _path_and_parents(selected, root)
+            paths |= _walk_directories(selected, max_depth)
+
+    names = [f"{os.path.basename(root) or root} (root)"]
+    sorted_paths = [root] + sorted(paths)
+    names += [os.path.relpath(path, root) for path in sorted_paths[1:]]
+
+    return names, sorted_paths
+
+
+def _walk_directories(base_path: str, max_depth: int) -> set[str]:
+    directories = set()
+
+    for dirpath, dirnames, _ in os.walk(base_path):
+        depth = dirpath[len(base_path) :].count(os.sep)
+        if depth >= max_depth:
+            dirnames.clear()
+            continue
+
+        dirnames[:] = [dir for dir in dirnames if not dir.startswith(".")]
+        directories.update(os.path.join(dirpath, dir) for dir in dirnames)
+
+    return directories
+
+
+def _path_and_parents(path: str, root: str) -> set[str]:
+    paths = set()
+
+    while path != root and path != os.path.dirname(path):
+        paths.add(path)
+        path = os.path.dirname(path)
+
+    return paths
+
+
+def _is_inside(path: str, root: str) -> bool:
+    return os.path.commonpath([path, root]) == root